feat: 合并 Docker Compose、标注表格优化与部署文档
将 platform + CVAT 合并为单文件 docker-compose.yml,完善 .env 与 init/dev_up 脚本; 新增 docs/DEPLOY.md 与更新 README 以支持新机器部署;含数据湖示例、车队地图、 紧凑表格 UI、ADAS det_7cls 路径与批次台账等近期改动。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
294
platform/as_platform/labeling/batch_index.py
Normal file
294
platform/as_platform/labeling/batch_index.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""批次索引:扫盘结果落库,列表页只查 DB(<200ms)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.data.core import get_pending_report, load_wf
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import BatchIndex, LabelingCampaign
|
||||
from as_platform.labeling.scope import enrich_batch_labels, load_dms_registry
|
||||
from as_platform.labeling.stage import STAGE_ALIASES, effective_stage
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def campaign_id_for_row(project: str, task: str, mode: str | None, batch: str, location: str) -> str:
|
||||
from as_platform.labeling.service import _campaign_id
|
||||
|
||||
return _campaign_id(project, task, mode, batch, location)
|
||||
|
||||
|
||||
def _batch_dict_to_fields(b: dict[str, Any], reg: dict) -> dict[str, Any] | None:
|
||||
if b.get("registry_only") and not b.get("batch"):
|
||||
return None
|
||||
row = enrich_batch_labels(b, reg)
|
||||
if row.get("registry_only"):
|
||||
pass
|
||||
raw_stage = row.get("stage")
|
||||
eff = effective_stage(raw_stage) or raw_stage or "raw_pool"
|
||||
allowed = (
|
||||
"raw_pool", "out_for_labeling", "returned", "labeling_submitted",
|
||||
"in_review", "review_approved", "review_rejected", "ingested",
|
||||
)
|
||||
if eff not in allowed and raw_stage not in allowed:
|
||||
return None
|
||||
project = row.get("project") or "dms"
|
||||
task = row.get("task") or ""
|
||||
mode = row.get("mode")
|
||||
batch = row.get("batch") or ""
|
||||
location = row.get("location") or "inbox"
|
||||
counts = row.get("counts") or {}
|
||||
cid = campaign_id_for_row(project, task, mode, batch, location)
|
||||
return {
|
||||
"campaign_id": cid,
|
||||
"project": project,
|
||||
"task": task or None,
|
||||
"mode": mode,
|
||||
"batch": batch,
|
||||
"pack": row.get("pack"),
|
||||
"location": location,
|
||||
"stage": eff,
|
||||
"batch_path": row.get("path"),
|
||||
"scope_key": row.get("scope_key"),
|
||||
"engineer": row.get("engineer"),
|
||||
"format": row.get("format"),
|
||||
"image_count": int(counts.get("images") or 0),
|
||||
"label_count": int(counts.get("labels") or 0),
|
||||
"has_meta": bool(row.get("has_meta")),
|
||||
"registry_only": bool(row.get("registry_only")),
|
||||
"next_cli": row.get("next_cli"),
|
||||
"domain": row.get("domain"),
|
||||
"domain_label": row.get("domain_label"),
|
||||
"task_label": row.get("task_label"),
|
||||
"mode_label": row.get("mode_label"),
|
||||
"labeling_profile": row.get("labeling_profile"),
|
||||
"export_default": row.get("export_default"),
|
||||
"ml_adapter": row.get("ml_adapter"),
|
||||
"indexed_at": _utcnow(),
|
||||
}
|
||||
|
||||
|
||||
def upsert_batch_dict(b: dict[str, Any], *, reg: dict | None = None) -> str | None:
|
||||
"""登记/单批更新时写入索引。"""
|
||||
reg = reg or load_dms_registry()
|
||||
fields = _batch_dict_to_fields(b, reg)
|
||||
if not fields:
|
||||
return None
|
||||
cid = fields["campaign_id"]
|
||||
fields["archived"] = False
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchIndex, cid)
|
||||
if rec:
|
||||
for k, v in fields.items():
|
||||
setattr(rec, k, v)
|
||||
else:
|
||||
db.add(BatchIndex(**fields))
|
||||
db.commit()
|
||||
return cid
|
||||
|
||||
|
||||
def archive_batch(campaign_id: str) -> dict[str, Any]:
|
||||
"""软删除:仅从工作台隐藏,不删磁盘数据。"""
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchIndex, campaign_id)
|
||||
if not rec:
|
||||
raise FileNotFoundError(campaign_id)
|
||||
if rec.archived:
|
||||
return {"ok": True, "campaign_id": campaign_id, "already_archived": True}
|
||||
if rec.stage != "raw_pool":
|
||||
raise ValueError("仅「待送标」批次可移除")
|
||||
camp = db.get(LabelingCampaign, campaign_id)
|
||||
if camp and camp.status not in ("not_opened", ""):
|
||||
raise ValueError("已开标的批次不可移除,请先在标注进度中处理")
|
||||
rec.archived = True
|
||||
rec.indexed_at = _utcnow()
|
||||
db.commit()
|
||||
return {"ok": True, "campaign_id": campaign_id}
|
||||
|
||||
|
||||
def sync_index_stage(campaign_id: str, stage: str) -> None:
|
||||
eff = effective_stage(stage) or stage
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchIndex, campaign_id)
|
||||
if rec:
|
||||
rec.stage = eff
|
||||
rec.indexed_at = _utcnow()
|
||||
db.commit()
|
||||
|
||||
|
||||
def index_count() -> int:
|
||||
with session_scope() as db:
|
||||
return db.query(BatchIndex).count()
|
||||
|
||||
|
||||
def index_is_empty() -> bool:
|
||||
return index_count() == 0
|
||||
|
||||
|
||||
def get_batch_by_campaign_id(campaign_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(BatchIndex, campaign_id)
|
||||
return rec.to_list_row() if rec else None
|
||||
|
||||
|
||||
def rebuild_batch_index(wf: dict | None = None) -> dict[str, Any]:
|
||||
"""全量扫盘并重建索引(刷新/首次加载/登记后批量同步)。"""
|
||||
from as_platform.labeling.service import _registry_fallback_batches
|
||||
|
||||
t0 = time.perf_counter()
|
||||
wf = wf or load_wf()
|
||||
reg = load_dms_registry()
|
||||
report = get_pending_report(wf)
|
||||
candidates: list[dict[str, Any]] = list(report.get("batches") or [])
|
||||
candidates.extend(_registry_fallback_batches(wf, reg))
|
||||
|
||||
seen: set[str] = set()
|
||||
upserted = 0
|
||||
with session_scope() as db:
|
||||
for b in candidates:
|
||||
fields = _batch_dict_to_fields(b, reg)
|
||||
if not fields:
|
||||
continue
|
||||
cid = fields["campaign_id"]
|
||||
seen.add(cid)
|
||||
rec = db.get(BatchIndex, cid)
|
||||
if rec and rec.archived:
|
||||
continue
|
||||
if rec:
|
||||
for k, v in fields.items():
|
||||
setattr(rec, k, v)
|
||||
else:
|
||||
db.add(BatchIndex(**fields))
|
||||
upserted += 1
|
||||
|
||||
if seen:
|
||||
db.query(BatchIndex).filter(
|
||||
BatchIndex.campaign_id.notin_(seen),
|
||||
BatchIndex.archived.is_(False),
|
||||
).delete(synchronize_session=False)
|
||||
db.commit()
|
||||
|
||||
elapsed_ms = round((time.perf_counter() - t0) * 1000)
|
||||
return {
|
||||
"ok": True,
|
||||
"count": upserted,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
"updated_at": report.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
def _expand_stage_filters_for_sql(stage_filters: list[str]) -> list[str]:
|
||||
"""将筛选阶段展开为索引表中的实际 stage 值(含 review_approved 等别名)。"""
|
||||
expanded: set[str] = set()
|
||||
for sf in stage_filters:
|
||||
expanded.add(sf)
|
||||
for alias, canonical in STAGE_ALIASES.items():
|
||||
if canonical == sf:
|
||||
expanded.add(alias)
|
||||
return list(expanded)
|
||||
|
||||
|
||||
def _index_query(db, *, stage_filters: list[str], q: str | None):
|
||||
from sqlalchemy import func, or_
|
||||
|
||||
from as_platform.config import IS_POSTGRES
|
||||
|
||||
query = db.query(BatchIndex).filter(
|
||||
BatchIndex.registry_only.is_(False),
|
||||
BatchIndex.archived.is_(False),
|
||||
)
|
||||
if stage_filters:
|
||||
query = query.filter(BatchIndex.stage.in_(_expand_stage_filters_for_sql(stage_filters)))
|
||||
text = (q or "").strip()
|
||||
if text:
|
||||
pattern = f"%{text}%"
|
||||
if IS_POSTGRES:
|
||||
query = query.filter(
|
||||
or_(
|
||||
BatchIndex.batch.ilike(pattern),
|
||||
BatchIndex.task.ilike(pattern),
|
||||
BatchIndex.project.ilike(pattern),
|
||||
)
|
||||
)
|
||||
else:
|
||||
lp = pattern.lower()
|
||||
query = query.filter(
|
||||
or_(
|
||||
func.lower(BatchIndex.batch).like(lp),
|
||||
func.lower(func.coalesce(BatchIndex.task, "")).like(lp),
|
||||
func.lower(BatchIndex.project).like(lp),
|
||||
)
|
||||
)
|
||||
return query
|
||||
|
||||
|
||||
def list_batches_from_index(
|
||||
*,
|
||||
stage: str | None = None,
|
||||
stages: list[str] | None = None,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
q: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
stage_filters = [s.strip() for s in (stages or []) if s and s.strip()]
|
||||
if stage and stage not in stage_filters:
|
||||
stage_filters.append(stage)
|
||||
|
||||
with session_scope() as db:
|
||||
base = _index_query(db, stage_filters=stage_filters, q=q)
|
||||
total = base.count()
|
||||
recs = (
|
||||
base.order_by(BatchIndex.indexed_at.desc())
|
||||
.offset(max(0, offset))
|
||||
.limit(max(1, limit))
|
||||
.all()
|
||||
)
|
||||
page = [rec.to_list_row() for rec in recs]
|
||||
latest_indexed = max((rec.indexed_at for rec in recs), default=None)
|
||||
|
||||
cids = [r["campaign_id"] for r in page if r.get("campaign_id")]
|
||||
|
||||
camp_map: dict[str, dict[str, Any]] = {}
|
||||
if cids:
|
||||
with session_scope() as db:
|
||||
camps = db.query(LabelingCampaign).filter(LabelingCampaign.id.in_(cids)).all()
|
||||
for c in camps:
|
||||
camp_map[c.id] = {
|
||||
"status": c.status,
|
||||
"assigned_to_user_id": c.assigned_to_user_id,
|
||||
"assigned_to_name": c.assigned_to_name,
|
||||
}
|
||||
|
||||
progress_stages = frozenset({"out_for_labeling", "in_progress"})
|
||||
out: list[dict[str, Any]] = []
|
||||
for row in page:
|
||||
cid = row.get("campaign_id")
|
||||
camp = camp_map.get(cid) if cid else None
|
||||
status = camp["status"] if camp else "not_opened"
|
||||
if camp:
|
||||
row["assigned_to_user_id"] = camp["assigned_to_user_id"]
|
||||
row["assigned_to_name"] = camp["assigned_to_name"]
|
||||
row["campaign_status"] = status
|
||||
eff_stage = row.get("stage") or ""
|
||||
if camp and (status in progress_stages or eff_stage in progress_stages):
|
||||
try:
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
|
||||
row.update(campaign_progress_summary(cid))
|
||||
except Exception:
|
||||
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
|
||||
out.append(row)
|
||||
|
||||
updated_at = latest_indexed.isoformat() if latest_indexed else None
|
||||
return {
|
||||
"items": out,
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"updated_at": updated_at,
|
||||
"source": "index",
|
||||
}
|
||||
@@ -70,6 +70,12 @@ def update_campaign_batch_meta_stage(camp: LabelingCampaign, stage: str) -> bool
|
||||
if camp.mode:
|
||||
meta.setdefault("mode", camp.mode)
|
||||
write_meta(batch_dir, meta)
|
||||
try:
|
||||
from as_platform.labeling.batch_index import sync_index_stage
|
||||
|
||||
sync_index_stage(camp.id, stage)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -107,7 +107,10 @@ def _labels_from_registry_profile(project: str, task: str, mode: str | None) ->
|
||||
prof = (load_labeling_registry().get("profiles") or {}).get(pk) or {}
|
||||
cvat_names = prof.get("cvat_labels")
|
||||
if cvat_names:
|
||||
return [{"name": str(n), "type": "cuboid"} for n in cvat_names]
|
||||
label_type = prof.get("cvat_label_type") or (
|
||||
"cuboid" if project == "adas" and task == "cuboid_7cls" else "rectangle"
|
||||
)
|
||||
return [{"name": str(n), "type": label_type} for n in cvat_names]
|
||||
return None
|
||||
|
||||
|
||||
@@ -126,6 +129,10 @@ def build_cvat_labels(
|
||||
if project == "adas":
|
||||
if task == "cuboid_7cls":
|
||||
return ADAS_CUBOID_7CLS_LABELS
|
||||
if task == "det_7cls":
|
||||
return [_rect_label(n) for n in [
|
||||
"pedestrian", "car", "truck", "bus", "motorcycle", "tricycle", "traffic cone",
|
||||
]]
|
||||
return ADAS_CUBOID_7CLS_LABELS
|
||||
|
||||
if project == "lane":
|
||||
@@ -145,4 +152,8 @@ def resolve_annotation_types(project: str, task: str | None = None, mode: str |
|
||||
}
|
||||
if project == "adas" and task == "cuboid_7cls":
|
||||
return ["cuboid"]
|
||||
if project == "adas" and task == "det_7cls":
|
||||
return ["bbox"]
|
||||
if project == "dms" and task == "adas":
|
||||
return ["bbox"]
|
||||
return mapping.get(project, ["bbox"])
|
||||
|
||||
@@ -108,66 +108,23 @@ def _registry_fallback_batches(wf: dict, reg: dict) -> list[dict[str, Any]]:
|
||||
def list_labeling_batches(
|
||||
*,
|
||||
stage: str | None = None,
|
||||
stages: list[str] | None = None,
|
||||
offset: int = 0,
|
||||
limit: int = 20,
|
||||
refresh: bool = False,
|
||||
q: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
wf = load_wf()
|
||||
report = get_pending_report(wf)
|
||||
reg = load_dms_registry()
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
allowed_stages = ("raw_pool", "out_for_labeling", "returned", "labeling_submitted", "in_review", "review_approved", "review_rejected")
|
||||
from as_platform.labeling.batch_index import (
|
||||
index_is_empty,
|
||||
list_batches_from_index,
|
||||
rebuild_batch_index,
|
||||
)
|
||||
|
||||
def _append(b: dict[str, Any]) -> None:
|
||||
if b.get("registry_only"):
|
||||
return
|
||||
raw_stage = b.get("stage")
|
||||
eff = effective_stage(raw_stage)
|
||||
if stage and not matches_stage_filter(raw_stage, stage):
|
||||
return
|
||||
if eff not in allowed_stages and raw_stage not in allowed_stages:
|
||||
return
|
||||
row = enrich_batch_labels(b, reg)
|
||||
row["stage"] = eff or raw_stage
|
||||
cid = _campaign_id(
|
||||
row["project"], row.get("task") or "", row.get("mode"), row["batch"], row.get("location") or "inbox"
|
||||
)
|
||||
key = f"{cid}"
|
||||
if key in seen:
|
||||
return
|
||||
seen.add(key)
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
status = camp.status if camp else "not_opened"
|
||||
if camp:
|
||||
row["assigned_to_user_id"] = camp.assigned_to_user_id
|
||||
row["assigned_to_name"] = camp.assigned_to_name
|
||||
row["campaign_id"] = cid
|
||||
row["campaign_status"] = status
|
||||
if camp and status in ("in_progress", "labeling_submitted"):
|
||||
try:
|
||||
from as_platform.labeling.progress import campaign_progress_summary
|
||||
|
||||
row.update(campaign_progress_summary(cid))
|
||||
except Exception:
|
||||
row.update({"total_tasks": 0, "completed_tasks": 0, "assigned_tasks": 0})
|
||||
items.append(row)
|
||||
|
||||
for b in report.get("batches", []):
|
||||
_append(b)
|
||||
|
||||
for b in _registry_fallback_batches(wf, reg):
|
||||
_append(b)
|
||||
|
||||
total = len(items)
|
||||
page = items[max(0, offset) : max(0, offset) + max(1, limit)]
|
||||
return {
|
||||
"items": page,
|
||||
"total": total,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"updated_at": report.get("updated_at"),
|
||||
}
|
||||
if refresh or index_is_empty():
|
||||
rebuild_batch_index()
|
||||
return list_batches_from_index(
|
||||
stage=stage, stages=stages, offset=offset, limit=limit, q=q,
|
||||
)
|
||||
|
||||
|
||||
def open_campaign(
|
||||
@@ -189,13 +146,14 @@ def open_campaign(
|
||||
|
||||
cvat = get_cvat_client()
|
||||
if not cvat.ping():
|
||||
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose -f docker-compose.yml -f docker-compose.cvat.yml up -d")
|
||||
raise ValueError("CVAT 标注引擎不可用,请执行: docker compose up -d")
|
||||
|
||||
cvat_labels = build_cvat_labels(project, task, mode, ann_types)
|
||||
cvat_task = cvat.create_task(name=cid, labels=cvat_labels)
|
||||
cvat_task_id = cvat_task.id
|
||||
cvat_job_url = cvat_task.job_url
|
||||
|
||||
batch_dir = None
|
||||
with session_scope() as db:
|
||||
camp = db.get(LabelingCampaign, cid)
|
||||
if not camp:
|
||||
@@ -241,6 +199,26 @@ def open_campaign(
|
||||
reg = load_dms_registry() if project == "dms" else None
|
||||
row = enrich_batch_labels(out, reg)
|
||||
row["stage"] = "out_for_labeling"
|
||||
try:
|
||||
from as_platform.labeling.batch_index import upsert_batch_dict
|
||||
|
||||
if batch_dir and batch_dir.is_dir():
|
||||
from as_platform.data.batch import enrich_batch
|
||||
|
||||
upsert_batch_dict(
|
||||
enrich_batch(
|
||||
batch_dir,
|
||||
project=project,
|
||||
task=task,
|
||||
pack=pack,
|
||||
batch=batch,
|
||||
location=location,
|
||||
),
|
||||
)
|
||||
else:
|
||||
upsert_batch_dict(row)
|
||||
except Exception:
|
||||
pass
|
||||
return row
|
||||
|
||||
|
||||
@@ -376,23 +354,15 @@ def list_labeling_assignees() -> dict[str, Any]:
|
||||
|
||||
|
||||
def _find_batch_for_campaign_id(campaign_id: str) -> dict[str, Any] | None:
|
||||
"""由确定性 campaign_id 反查 pending / registry 批次行。"""
|
||||
wf = load_wf()
|
||||
reg = load_dms_registry()
|
||||
candidates: list[dict[str, Any]] = []
|
||||
report = get_pending_report(wf)
|
||||
candidates.extend(report.get("batches") or [])
|
||||
candidates.extend(_registry_fallback_batches(wf, reg))
|
||||
for b in candidates:
|
||||
cid = _campaign_id(
|
||||
b.get("project") or "dms",
|
||||
b.get("task") or "",
|
||||
b.get("mode"),
|
||||
b.get("batch") or "",
|
||||
b.get("location") or "inbox",
|
||||
)
|
||||
if cid == campaign_id:
|
||||
return b
|
||||
"""由 campaign_id 反查批次行(优先读索引)。"""
|
||||
from as_platform.labeling.batch_index import get_batch_by_campaign_id, index_is_empty, rebuild_batch_index
|
||||
|
||||
row = get_batch_by_campaign_id(campaign_id)
|
||||
if row:
|
||||
return row
|
||||
if index_is_empty():
|
||||
rebuild_batch_index()
|
||||
return get_batch_by_campaign_id(campaign_id)
|
||||
return None
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user