feat: 合并 Docker Compose、标注表格优化与部署文档

将 platform + CVAT 合并为单文件 docker-compose.yml,完善 .env 与 init/dev_up 脚本;
新增 docs/DEPLOY.md 与更新 README 以支持新机器部署;含数据湖示例、车队地图、
紧凑表格 UI、ADAS det_7cls 路径与批次台账等近期改动。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-06-16 17:06:31 +08:00
parent 0b8ade048e
commit 483e027482
117 changed files with 5933 additions and 1499 deletions

View File

@@ -0,0 +1,294 @@
"""批次索引:扫盘结果落库,列表页只查 DB<200ms"""
from __future__ import annotations
import time
from datetime import datetime, timezone
from typing import Any
from as_platform.data.core import get_pending_report, load_wf
from as_platform.db.engine import session_scope
from as_platform.db.models import BatchIndex, LabelingCampaign
from as_platform.labeling.scope import enrich_batch_labels, load_dms_registry
from as_platform.labeling.stage import STAGE_ALIASES, effective_stage
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
def campaign_id_for_row(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
from as_platform.labeling.service import _campaign_id
return _campaign_id(project, task, mode, batch, location)
def _batch_dict_to_fields(b: dict[str, Any], reg: dict) -> dict[str, Any] | None:
if b.get("registry_only") and not b.get("batch"):
return None
row = enrich_batch_labels(b, reg)
if row.get("registry_only"):
pass
raw_stage = row.get("stage")
eff = effective_stage(raw_stage) or raw_stage or "raw_pool"
allowed = (
"raw_pool", "out_for_labeling", "returned", "labeling_submitted",
"in_review", "review_approved", "review_rejected", "ingested",
)
if eff not in allowed and raw_stage not in allowed:
return None
project = row.get("project") or "dms"
task = row.get("task") or ""
mode = row.get("mode")
batch = row.get("batch") or ""
location = row.get("location") or "inbox"
counts = row.get("counts") or {}
cid = campaign_id_for_row(project, task, mode, batch, location)
return {
"campaign_id": cid,
"project": project,
"task": task or None,
"mode": mode,
"batch": batch,
"pack": row.get("pack"),
"location": location,
"stage": eff,
"batch_path": row.get("path"),
"scope_key": row.get("scope_key"),
"engineer": row.get("engineer"),
"format": row.get("format"),
"image_count": int(counts.get("images") or 0),
"label_count": int(counts.get("labels") or 0),
"has_meta": bool(row.get("has_meta")),
"registry_only": bool(row.get("registry_only")),
"next_cli": row.get("next_cli"),
"domain": row.get("domain"),
"domain_label": row.get("domain_label"),
"task_label": row.get("task_label"),
"mode_label": row.get("mode_label"),
"labeling_profile": row.get("labeling_profile"),
"export_default": row.get("export_default"),
"ml_adapter": row.get("ml_adapter"),
"indexed_at": _utcnow(),
}
def upsert_batch_dict(b: dict[str, Any], *, reg: dict | None = None) -> str | None:
"""登记/单批更新时写入索引。"""
reg = reg or load_dms_registry()
fields = _batch_dict_to_fields(b, reg)
if not fields:
return None
cid = fields["campaign_id"]
fields["archived"] = False
with session_scope() as db:
rec = db.get(BatchIndex, cid)
if rec:
for k, v in fields.items():
setattr(rec, k, v)
else:
db.add(BatchIndex(**fields))
db.commit()
return cid
def archive_batch(campaign_id: str) -> dict[str, Any]:
"""软删除:仅从工作台隐藏,不删磁盘数据。"""
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
if not rec:
raise FileNotFoundError(campaign_id)
if rec.archived:
return {"ok": True, "campaign_id": campaign_id, "already_archived": True}
if rec.stage != "raw_pool":
raise ValueError("仅「待送标」批次可移除")
camp = db.get(LabelingCampaign, campaign_id)
if camp and camp.status not in ("not_opened", ""):
raise ValueError("已开标的批次不可移除,请先在标注进度中处理")
rec.archived = True
rec.indexed_at = _utcnow()
db.commit()
return {"ok": True, "campaign_id": campaign_id}
def sync_index_stage(campaign_id: str, stage: str) -> None:
eff = effective_stage(stage) or stage
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
if rec:
rec.stage = eff
rec.indexed_at = _utcnow()
db.commit()
def index_count() -> int:
with session_scope() as db:
return db.query(BatchIndex).count()
def index_is_empty() -> bool:
return index_count() == 0
def get_batch_by_campaign_id(campaign_id: str) -> dict[str, Any] | None:
with session_scope() as db:
rec = db.get(BatchIndex, campaign_id)
return rec.to_list_row() if rec else None
def rebuild_batch_index(wf: dict | None = None) -> dict[str, Any]:
"""全量扫盘并重建索引(刷新/首次加载/登记后批量同步)。"""
from as_platform.labeling.service import _registry_fallback_batches
t0 = time.perf_counter()
wf = wf or load_wf()
reg = load_dms_registry()
report = get_pending_report(wf)
candidates: list[dict[str, Any]] = list(report.get("batches") or [])
candidates.extend(_registry_fallback_batches(wf, reg))
seen: set[str] = set()
upserted = 0
with session_scope() as db:
for b in candidates:
fields = _batch_dict_to_fields(b, reg)
if not fields:
continue
cid = fields["campaign_id"]
seen.add(cid)
rec = db.get(BatchIndex, cid)
if rec and rec.archived:
continue
if rec:
for k, v in fields.items():
setattr(rec, k, v)
else:
db.add(BatchIndex(**fields))
upserted += 1
if seen:
db.query(BatchIndex).filter(
BatchIndex.campaign_id.notin_(seen),
BatchIndex.archived.is_(False),
).delete(synchronize_session=False)
db.commit()
elapsed_ms = round((time.perf_counter() - t0) * 1000)
return {
"ok": True,
"count": upserted,
"elapsed_ms": elapsed_ms,
"updated_at": report.get("updated_at"),
}
def _expand_stage_filters_for_sql(stage_filters: list[str]) -> list[str]:
"""将筛选阶段展开为索引表中的实际 stage 值(含 review_approved 等别名)。"""
expanded: set[str] = set()
for sf in stage_filters:
expanded.add(sf)
for alias, canonical in STAGE_ALIASES.items():
if canonical == sf:
expanded.add(alias)
return list(expanded)
def _index_query(db, *, stage_filters: list[str], q: str | None):
from sqlalchemy import func, or_
from as_platform.config import IS_POSTGRES
query = db.query(BatchIndex).filter(
BatchIndex.registry_only.is_(False),
BatchIndex.archived.is_(False),
)
if stage_filters:
query = query.filter(BatchIndex.stage.in_(_expand_stage_filters_for_sql(stage_filters)))
text = (q or "").strip()
if text:
pattern = f"%{text}%"
if IS_POSTGRES:
query = query.filter(
or_(
BatchIndex.batch.ilike(pattern),
BatchIndex.task.ilike(pattern),
BatchIndex.project.ilike(pattern),
)
)
else:
lp = pattern.lower()
query = query.filter(
or_(
func.lower(BatchIndex.batch).like(lp),
func.lower(func.coalesce(BatchIndex.task, "")).like(lp),
func.lower(BatchIndex.project).like(lp),
)
)
return query
def list_batches_from_index(
*,
stage: str | None = None,
stages: list[str] | None = None,
offset: int = 0,
limit: int = 20,
q: str | None = None,
) -> dict[str, Any]:
stage_filters = [s.strip() for s in (stages or []) if s and s.strip()]
if stage and stage not in stage_filters:
stage_filters.append(stage)
with session_scope() as db:
base = _index_query(db, stage_filters=stage_filters, q=q)
total = base.count()
recs = (
base.order_by(BatchIndex.indexed_at.desc())
.offset(max(0, offset))
.limit(max(1, limit))
.all()
)
page = [rec.to_list_row() for rec in recs]
latest_indexed = max((rec.indexed_at for rec in recs), default=None)
cids = [r["campaign_id"] for r in page if r.get("campaign_id")]
camp_map: dict[str, dict[str, Any]] = {}
if cids:
with session_scope() as db:
camps = db.query(LabelingCampaign).filter(LabelingCampaign.id.in_(cids)).all()
for c in camps:
camp_map[c.id] = {
"status": c.status,
"assigned_to_user_id": c.assigned_to_user_id,
"assigned_to_name": c.assigned_to_name,
}
progress_stages = frozenset({"out_for_labeling", "in_progress"})
out: list[dict[str, Any]] = []
for row in page:
cid = row.get("campaign_id")
camp = camp_map.get(cid) if cid else None
status = camp["status"] if camp else "not_opened"
if camp:
row["assigned_to_user_id"] = camp["assigned_to_user_id"]
row["assigned_to_name"] = camp["assigned_to_name"]
row["campaign_status"] = status
eff_stage = row.get("stage") or ""
if camp and (status in progress_stages or eff_stage in progress_stages):
try:
from as_platform.labeling.progress import campaign_progress_summary
row.update(campaign_progress_summary(cid))
except Exception:
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
out.append(row)
updated_at = latest_indexed.isoformat() if latest_indexed else None
return {
"items": out,
"total": total,
"offset": offset,
"limit": limit,
"updated_at": updated_at,
"source": "index",
}

View File

@@ -70,6 +70,12 @@ def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool
if camp.mode:
meta.setdefault("mode", camp.mode)
write_meta(batch_dir, meta)
try:
from as_platform.labeling.batch_index import sync_index_stage
sync_index_stage(camp.id, stage)
except Exception:
pass
return True

View File

@@ -107,7 +107,10 @@ def _labels_from_registry_profile(project: str, task: str, mode: str | None) ->
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
cvat_names = prof.get("cvat_labels")
if cvat_names:
return [{"name": str(n), "type": "cuboid"} for n in cvat_names]
label_type = prof.get("cvat_label_type") or (
"cuboid" if project == "adas" and task == "cuboid_7cls" else "rectangle"
)
return [{"name": str(n), "type": label_type} for n in cvat_names]
return None
@@ -126,6 +129,10 @@ def build_cvat_labels(
if project == "adas":
if task == "cuboid_7cls":
return ADAS_CUBOID_7CLS_LABELS
if task == "det_7cls":
return [_rect_label(n) for n in [
"pedestrian", "car", "truck", "bus", "motorcycle", "tricycle", "traffic cone",
]]
return ADAS_CUBOID_7CLS_LABELS
if project == "lane":
@@ -145,4 +152,8 @@ def resolve_annotation_types(project: str, task: str | None = None, mode: str |
}
if project == "adas" and task == "cuboid_7cls":
return ["cuboid"]
if project == "adas" and task == "det_7cls":
return ["bbox"]
if project == "dms" and task == "adas":
return ["bbox"]
return mapping.get(project, ["bbox"])

View File

@@ -108,66 +108,23 @@ def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
def list_labeling_batches(
*,
stage: str | None = None,
stages: list[str] | None = None,
offset: int = 0,
limit: int = 20,
refresh: bool = False,
q: str | None = None,
) -> dict[str, Any]:
wf = load_wf()
report = get_pending_report(wf)
reg = load_dms_registry()
items: list[dict[str, Any]] = []
seen: set[str] = set()
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
from as_platform.labeling.batch_index import (
index_is_empty,
list_batches_from_index,
rebuild_batch_index,
)
def _append(b: dict[str, Any]) -> None:
if b.get("registry_only"):
return
raw_stage = b.get("stage")
eff = effective_stage(raw_stage)
if stage and not matches_stage_filter(raw_stage, stage):
return
if eff not in allowed_stages and raw_stage not in allowed_stages:
return
row = enrich_batch_labels(b, reg)
row["stage"] = eff or raw_stage
cid = _campaign_id(
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
)
key = f"{cid}"
if key in seen:
return
seen.add(key)
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
status = camp.status if camp else "not_opened"
if camp:
row["assigned_to_user_id"] = camp.assigned_to_user_id
row["assigned_to_name"] = camp.assigned_to_name
row["campaign_id"] = cid
row["campaign_status"] = status
if camp and status in ("in_progress", "labeling_submitted"):
try:
from as_platform.labeling.progress import campaign_progress_summary
row.update(campaign_progress_summary(cid))
except Exception:
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
items.append(row)
for b in report.get("batches", []):
_append(b)
for b in _registry_fallback_batches(wf, reg):
_append(b)
total = len(items)
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
return {
"items": page,
"total": total,
"offset": offset,
"limit": limit,
"updated_at": report.get("updated_at"),
}
if refresh or index_is_empty():
rebuild_batch_index()
return list_batches_from_index(
stage=stage, stages=stages, offset=offset, limit=limit, q=q,
)
def open_campaign(
@@ -189,13 +146,14 @@ def open_campaign(
cvat = get_cvat_client()
if not cvat.ping():
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose -f docker-compose.yml -f docker-compose.cvat.yml up -d")
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose up -d")
cvat_labels = build_cvat_labels(project, task, mode, ann_types)
cvat_task = cvat.create_task(name=cid, labels=cvat_labels)
cvat_task_id = cvat_task.id
cvat_job_url = cvat_task.job_url
batch_dir = None
with session_scope() as db:
camp = db.get(LabelingCampaign, cid)
if not camp:
@@ -241,6 +199,26 @@ def open_campaign(
reg = load_dms_registry() if project == "dms" else None
row = enrich_batch_labels(out, reg)
row["stage"] = "out_for_labeling"
try:
from as_platform.labeling.batch_index import upsert_batch_dict
if batch_dir and batch_dir.is_dir():
from as_platform.data.batch import enrich_batch
upsert_batch_dict(
enrich_batch(
batch_dir,
project=project,
task=task,
pack=pack,
batch=batch,
location=location,
),
)
else:
upsert_batch_dict(row)
except Exception:
pass
return row
@@ -376,23 +354,15 @@ def list_labeling_assignees() -> dict[str, Any]:
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
"""确定性 campaign_id 反查 pending / registry 批次行。"""
wf = load_wf()
reg = load_dms_registry()
candidates: list[dict[str, Any]] = []
report = get_pending_report(wf)
candidates.extend(report.get("batches") or [])
candidates.extend(_registry_fallback_batches(wf, reg))
for b in candidates:
cid = _campaign_id(
b.get("project") or "dms",
b.get("task") or "",
b.get("mode"),
b.get("batch") or "",
b.get("location") or "inbox",
)
if cid == campaign_id:
return b
"""由 campaign_id 反查批次行(优先读索引)"""
from as_platform.labeling.batch_index import get_batch_by_campaign_id, index_is_empty, rebuild_batch_index
row = get_batch_by_campaign_id(campaign_id)
if row:
return row
if index_is_empty():
rebuild_batch_index()
return get_batch_by_campaign_id(campaign_id)
return None