"""扫描 inbox / 数据湖目录，与批次台账对齐。""" from __future__ import annotations from datetime import datetime, timezone from pathlib import Path from typing import Any from as_platform.data.core import load_wf, proj_root, register_batch from as_platform.db.engine import session_scope from as_platform.db.models import BatchDelivery, BatchIndex, User from as_platform.deliveries.service import _new_delivery_id, _normalize_task def _utcnow() -> datetime: return datetime.now(timezone.utc) def _dir_mtime_iso(path: Path) -> str | None: try: ts = path.stat().st_mtime return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d") except OSError: return None def _scan_project_inbox(project: str, wf: dict | None = None) -> list[dict[str, Any]]: from as_platform.data.batch import count_images, count_label_files, dms_has_labels wf = wf or load_wf() root = proj_root(wf, project) inbox = root / "inbox" if not inbox.is_dir(): return [] with session_scope() as db: deliveries = { (r.project, r.task or "", r.mode or "", r.batch_name): r for r in db.query(BatchDelivery).filter(BatchDelivery.project == project).all() } indexed = { (r.task or "", r.batch) for r in db.query(BatchIndex).filter( BatchIndex.project == project, BatchIndex.archived.is_(False), ).all() } items: list[dict[str, Any]] = [] for task_dir in sorted(inbox.iterdir()): if not task_dir.is_dir(): continue for batch_dir in sorted(task_dir.iterdir()): if not batch_dir.is_dir(): continue task_name = task_dir.name batch_name = batch_dir.name img_count = count_images(batch_dir) if not img_count and (batch_dir / "images").is_dir(): img_count = count_images(batch_dir / "images") lbl_count = count_label_files(batch_dir / "labels") if (batch_dir / "labels").is_dir() else 0 has_labels = lbl_count > 0 or dms_has_labels(batch_dir) stage_hint = "returned" if has_labels and lbl_count > 0 else "raw_pool" key = (project, task_name, "", batch_name) delivery = deliveries.get(key) in_index = (task_name, batch_name) in indexed items.append({ "project": project, "task": task_name, "mode": None, "batch": batch_name, "batch_name": batch_name, "path": str(batch_dir), "data_path": str(batch_dir), "images": img_count, "labels": lbl_count, "has_labels": has_labels, "stage_hint": stage_hint, "source_type": "inbox_scan", "delivery_id": delivery.id if delivery else None, "delivery_status": delivery.status if delivery else None, "in_ledger": delivery is not None, "in_workbench": in_index, "collection_start": delivery.collection_start if delivery else _dir_mtime_iso(batch_dir), "collection_end": delivery.collection_end if delivery else None, "created_at": delivery.created_at.isoformat() if delivery and delivery.created_at else None, "needs_ledger": delivery is None, "needs_workbench": not in_index, }) return items def scan_delivery_sources(*, projects: list[str] | None = None) -> dict[str, Any]: """扫描 inbox，返回与台账、工作台对齐状态。""" projs = projects or ["dms", "adas", "lane"] wf = load_wf() items: list[dict[str, Any]] = [] for p in projs: items.extend(_scan_project_inbox(p, wf)) needs_ledger = sum(1 for i in items if i.get("needs_ledger")) needs_workbench = sum(1 for i in items if i.get("needs_workbench")) return { "items": items, "count": len(items), "needs_ledger": needs_ledger, "needs_workbench": needs_workbench, "scanned_at": _utcnow().isoformat(), } def register_scanned_to_ledger( items: list[dict[str, Any]], user: User, *, sync_workbench: bool = True, ) -> dict[str, Any]: """将扫描结果登记到台账；已在 inbox 的批次直接标为 in_lake 并同步工作台。""" created = 0 updated = 0 synced = 0 out_items: list[dict[str, Any]] = [] for raw in items: project = (raw.get("project") or "dms").strip() task = _normalize_task(project, raw.get("task")) mode = (raw.get("mode") or "").strip() or None batch_name = (raw.get("batch_name") or raw.get("batch") or "").strip() data_path = (raw.get("data_path") or raw.get("path") or "").strip() if not batch_name or not data_path: continue if not Path(data_path).is_dir(): continue stage_hint = raw.get("stage_hint") or "raw_pool" collection_start = (raw.get("collection_start") or "").strip() or _dir_mtime_iso(Path(data_path)) collection_end = (raw.get("collection_end") or "").strip() or None estimated = raw.get("images") if estimated is None: estimated = raw.get("estimated_count") with session_scope() as db: rec = ( db.query(BatchDelivery) .filter_by(project=project, task=task, mode=mode, batch_name=batch_name) .first() ) if not rec: rec = BatchDelivery( id=_new_delivery_id(), project=project, task=task, mode=mode, batch_name=batch_name, source_type=(raw.get("source_type") or "inbox_scan"), collection_start=collection_start, collection_end=collection_end, data_path=data_path, estimated_count=int(estimated) if estimated not in (None, "") else None, status="in_lake", inbox_path=data_path, owner_user_id=user.id, owner_name=user.name, submitted_by_user_id=user.id, submitted_by_name=user.name, ) db.add(rec) created += 1 else: if rec.status in ("draft", "rejected", "ingest_failed"): rec.status = "in_lake" if not rec.inbox_path: rec.inbox_path = data_path if not rec.data_path: rec.data_path = data_path if collection_start and not rec.collection_start: rec.collection_start = collection_start if estimated not in (None, "") and not rec.estimated_count: rec.estimated_count = int(estimated) if not rec.source_type: rec.source_type = "inbox_scan" rec.updated_at = _utcnow() updated += 1 db.flush() out_items.append(rec.to_dict()) if sync_workbench and stage_hint in ("raw_pool", "returned"): try: register_batch( None, project, task, batch_name, stage=stage_hint, location="inbox", ) synced += 1 except Exception: pass return { "ok": True, "created": created, "updated": updated, "synced_workbench": synced, "items": out_items, } def bridge_delivery_to_workbench(delivery_id: str) -> dict[str, Any]: """台账 in_lake 后同步到送标工作台索引。""" with session_scope() as db: rec = db.get(BatchDelivery, delivery_id) if not rec: raise ValueError("送标申请不存在") if rec.status != "in_lake": raise ValueError(f"当前状态不可同步工作台: {rec.status}") project = rec.project task = rec.task batch_name = rec.batch_name inbox_path = rec.inbox_path or rec.data_path stage = "raw_pool" if inbox_path: labels_dir = Path(inbox_path) / "labels" if labels_dir.is_dir() and any(labels_dir.iterdir()): stage = "returned" result = register_batch(None, project, task, batch_name, stage=stage, location="inbox") return {"ok": True, "delivery_id": delivery_id, "batch": result.get("batch")}