feat: initial HSAP platform

Huaxu Sentinel Active Safety Platform with embedded algorithm code,
Docker Compose setup, and vendored dataset scaffolds for clone-and-run.

Co-authored-by: Cursor <cursoragent@cursor.com>

platform/as_platform/data/__init__.py

@@ -0,0 +1,37 @@
+from as_platform.data.batch import META_FILENAME
+from as_platform.data.core import (
+    get_catalog,
+    get_pending_report,
+    load_wf,
+    proj_root,
+    register_batch,
+    resolve_pack,
+    resolve_pack_dir,
+)
+from as_platform.data.ingest import inspect_uploaded_dataset
+from as_platform.data.lake import (
+    analyze_uploaded_candidate,
+    create_uploaded_candidate,
+    get_candidate,
+    link_candidate_analysis_job,
+    list_candidates,
+    write_candidate_upload,
+)
+
+__all__ = [
+    "META_FILENAME",
+    "get_pending_report",
+    "get_catalog",
+    "register_batch",
+    "load_wf",
+    "proj_root",
+    "resolve_pack",
+    "resolve_pack_dir",
+    "inspect_uploaded_dataset",
+    "create_uploaded_candidate",
+    "write_candidate_upload",
+    "list_candidates",
+    "get_candidate",
+    "link_candidate_analysis_job",
+    "analyze_uploaded_candidate",
+]

platform/as_platform/data/batch.py

@@ -0,0 +1,161 @@
+"""批次元数据 batch.meta.yaml 与目录结构推断。"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+META_FILENAME = "batch.meta.yaml"
+SCHEMA = "huaxu-batch-v1"
+IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
+
+
+def count_images(dir_path: Path) -> int:
+    if not dir_path.is_dir():
+        return 0
+    n = 0
+    for p in dir_path.rglob("*"):
+        if p.is_file() and p.suffix in IMG_EXTS:
+            n += 1
+    return n
+
+
+def count_label_files(dir_path: Path) -> int:
+    if not dir_path.is_dir():
+        return 0
+    n = 0
+    for p in dir_path.rglob("*"):
+        if p.is_file() and p.suffix.lower() in (".txt", ".xml"):
+            n += 1
+    return n
+
+
+def dms_has_images(batch_dir: Path) -> bool:
+    for sub in ("images", "images/train"):
+        if count_images(batch_dir / sub) > 0:
+            return True
+    return False
+
+
+def dms_has_labels(batch_dir: Path) -> bool:
+    for sub in ("labels", "labels/train"):
+        d = batch_dir / sub
+        if d.is_dir() and count_label_files(d) > 0:
+            return True
+    return False
+
+
+def infer_dms_stage(batch_dir: Path) -> str:
+    has_img = dms_has_images(batch_dir)
+    has_lab = dms_has_labels(batch_dir)
+    if has_img and has_lab:
+        return "returned"
+    if has_img:
+        return "raw_pool"
+    return "raw_pool"
+
+
+def infer_lane_stage(path: Path) -> str:
+    if (path / "list" / "train_gt.txt").is_file():
+        return "ingested"
+    if (path / "train_val_gt.txt").is_file():
+        return "returned"
+    if any(path.glob("**/train_val_gt.txt")):
+        return "returned"
+    if count_images(path) > 0:
+        return "raw_pool"
+    return "raw_pool"
+
+
+def read_meta(batch_dir: Path) -> dict[str, Any] | None:
+    p = batch_dir / META_FILENAME
+    if not p.is_file():
+        return None
+    try:
+        data = yaml.safe_load(p.read_text(encoding="utf-8"))
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+
+
+def write_meta(batch_dir: Path, data: dict[str, Any]) -> Path:
+    batch_dir.mkdir(parents=True, exist_ok=True)
+    data.setdefault("schema", SCHEMA)
+    p = batch_dir / META_FILENAME
+    p.write_text(
+        yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False),
+        encoding="utf-8",
+    )
+    return p
+
+
+def enrich_batch(
+    batch_dir: Path,
+    *,
+    project: str,
+    task: str | None = None,
+    pack: str | None = None,
+    batch: str,
+    location: str,
+) -> dict[str, Any]:
+    meta = read_meta(batch_dir) or {}
+    if project == "dms":
+        stage = meta.get("stage") or infer_dms_stage(batch_dir)
+        img_n = meta.get("counts", {}).get("images")
+        lab_n = meta.get("counts", {}).get("labels")
+        if img_n is None:
+            img_n = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
+        if lab_n is None:
+            lab_n = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
+        fmt = meta.get("format", "yolo")
+    else:
+        stage = meta.get("stage") or infer_lane_stage(batch_dir)
+        img_n = meta.get("counts", {}).get("images") or count_images(batch_dir)
+        lab_n = meta.get("counts", {}).get("labels")
+        fmt = meta.get("format", "ufld_archive")
+        if (batch_dir / "list" / "train_gt.txt").is_file():
+            try:
+                lab_n = sum(1 for _ in (batch_dir / "list" / "train_gt.txt").open(encoding="utf-8"))
+            except OSError:
+                lab_n = lab_n or 0
+
+    return {
+        "project": project,
+        "task": task or meta.get("task"),
+        "pack": pack or meta.get("pack"),
+        "batch": batch,
+        "stage": stage,
+        "location": location,
+        "path": str(batch_dir.resolve()),
+        "engineer": meta.get("engineer"),
+        "format": fmt,
+        "counts": {"images": img_n, "labels": lab_n},
+        "has_meta": bool(meta),
+        "next_cli": suggest_cli(project, task, pack, batch, stage, location),
+    }
+
+
+def suggest_cli(
+    project: str,
+    task: str | None,
+    pack: str | None,
+    batch: str,
+    stage: str,
+    location: str,
+) -> str:
+    if project == "dms":
+        p = pack or "dms_v2"
+        t = task or "<task>"
+        if location == "inbox" and stage == "returned":
+            return f"python as.py build dms {t} --pack {p} --batch {batch}"
+        if location == "sources" and stage == "returned":
+            return f"python as.py build dms {t} --pack {p} --all-sources"
+        if stage == "raw_pool":
+            return f"# 送标完成后放入 labels，或: python as.py register-batch dms {t} --batch {batch} --stage returned"
+        return f"python as.py build dms {t} --pack {p}"
+    if stage == "returned":
+        return "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD"
+    if stage == "ingested":
+        return f"python as.py enable lane {pack or '<pack>'} && python as.py build lane"
+    return "python as.py add lane --src <path> --engineer <name> --date YYYYMMDD"

platform/as_platform/data/catalog_cache.py

@@ -0,0 +1,246 @@
+"""Catalog cache: memory/disk + cheap directory-change invalidation."""
+from __future__ import annotations
+
+import csv
+import json
+import os
+import time
+from pathlib import Path
+from typing import Any
+
+from as_platform.config import WORKSPACE
+
+CATALOG_CACHE_FILE = WORKSPACE / "manifests" / "catalog_cache.json"
+CATALOG_CACHE_TTL_SEC = int(os.environ.get("AS_CATALOG_CACHE_TTL_SEC", "300"))
+CATALOG_USE_REPORTS = os.environ.get("AS_CATALOG_USE_REPORTS", "1").lower() in ("1", "true", "yes")
+CATALOG_CACHE_VERSION = 3
+
+REPORTS_DIR = WORKSPACE / "reports"
+DMS_SUMMARY_CSV = REPORTS_DIR / "dms_task_image_summary.csv"
+DMS_CLASS_CSV = REPORTS_DIR / "dms_task_class_image_counts.csv"
+
+_CATALOG_MEM_CACHE: dict[str, Any] | None = None
+
+
+def invalidate_catalog_cache() -> None:
+    global _CATALOG_MEM_CACHE
+    _CATALOG_MEM_CACHE = None
+    if CATALOG_CACHE_FILE.is_file():
+        try:
+            CATALOG_CACHE_FILE.unlink()
+        except OSError:
+            pass
+
+
+def _dir_fingerprint(path: Path, *, scan_children: bool = True) -> dict[str, Any]:
+    if not path.exists():
+        return {"path": str(path), "missing": True}
+    try:
+        st = path.stat()
+        fp: dict[str, Any] = {
+            "path": str(path),
+            "mtime_ns": st.st_mtime_ns,
+            "size": st.st_size,
+        }
+        if scan_children and path.is_dir():
+            children: list[dict[str, Any]] = []
+            try:
+                with os.scandir(path) as it:
+                    for entry in it:
+                        if entry.name.startswith("."):
+                            continue
+                        try:
+                            est = entry.stat(follow_symlinks=False)
+                            children.append(
+                                {
+                                    "name": entry.name,
+                                    "mtime_ns": est.st_mtime_ns,
+                                    "is_dir": entry.is_dir(follow_symlinks=False),
+                                }
+                            )
+                        except OSError:
+                            children.append({"name": entry.name, "error": True})
+            except OSError:
+                fp["scan_error"] = True
+            children.sort(key=lambda x: x.get("name", ""))
+            fp["children"] = children
+        return fp
+    except OSError:
+        return {"path": str(path), "error": True}
+
+
+def build_catalog_signature(wf: dict, proj_root_fn) -> dict[str, Any]:
+    """Cheap signature: config files + inbox/pack directory mtimes (auto-invalidate on drop)."""
+    from as_platform.data.core import _pack_registry_path, load_pack_registry
+
+    files: list[dict[str, Any]] = []
+    for rel in ("workflow.registry.yaml",):
+        p = WORKSPACE / rel
+        try:
+            st = p.stat()
+            files.append({"path": str(p), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
+        except FileNotFoundError:
+            files.append({"path": str(p), "missing": True})
+
+    dirs: list[dict[str, Any]] = []
+    for pname in ("dms", "lane"):
+        root = proj_root_fn(wf, pname)
+        dirs.append(_dir_fingerprint(root, scan_children=False))
+        dirs.append(_dir_fingerprint(root / "inbox"))
+        try:
+            packs_reg = load_pack_registry(pname, root, wf)
+        except (FileNotFoundError, json.JSONDecodeError, OSError):
+            packs_reg = {"packs": []}
+        dirs.append(_dir_fingerprint(root / "packs", scan_children=False))
+        for p in packs_reg.get("packs", []):
+            pack_path = root / p.get("path", p.get("name", ""))
+            dirs.append(_dir_fingerprint(pack_path, scan_children=False))
+        if pname == "dms":
+            for cfg_path in (
+                root / wf["projects"]["dms"]["registry"],
+                root / "manifests" / "dataset_class_summary.txt",
+                _pack_registry_path("dms", root, wf),
+            ):
+                try:
+                    st = cfg_path.stat()
+                    files.append({"path": str(cfg_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
+                except FileNotFoundError:
+                    files.append({"path": str(cfg_path), "missing": True})
+            reg_path = root / wf["projects"]["dms"]["registry"]
+            if reg_path.is_file():
+                import yaml
+
+                reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
+                for task in (reg.get("tasks") or {}).keys():
+                    dirs.append(_dir_fingerprint(root / "inbox" / task))
+        if pname == "lane":
+            dirs.append(_dir_fingerprint(_pack_registry_path("lane", root, wf), scan_children=False))
+
+    for csv_path in (DMS_SUMMARY_CSV, DMS_CLASS_CSV):
+        try:
+            st = csv_path.stat()
+            files.append({"path": str(csv_path), "mtime_ns": st.st_mtime_ns, "size": st.st_size})
+        except FileNotFoundError:
+            files.append({"path": str(csv_path), "missing": True})
+
+    return {"files": files, "dirs": dirs}
+
+
+def load_disk_cache() -> dict[str, Any] | None:
+    if not CATALOG_CACHE_FILE.is_file():
+        return None
+    try:
+        return json.loads(CATALOG_CACHE_FILE.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return None
+
+
+def save_disk_cache(payload: dict[str, Any]) -> None:
+    CATALOG_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    CATALOG_CACHE_FILE.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+
+
+def _catalog_has_empty_bbox(catalog: dict[str, Any]) -> bool:
+    for task in (catalog.get("dms") or {}).values():
+        for pack in task.get("packs") or []:
+            boxes = int(pack.get("total_boxes") or 0)
+            pts = pack.get("bbox_points") or []
+            if boxes > 0 and not pts:
+                return True
+    return False
+
+
+def get_cached_catalog(signature: dict[str, Any], *, refresh: bool = False) -> tuple[dict[str, Any] | None, dict[str, Any]]:
+    """Return (catalog_data, meta). meta describes cache hit/miss."""
+    global _CATALOG_MEM_CACHE
+    now = time.time()
+    meta: dict[str, Any] = {"cached": False, "source": "scan"}
+
+    if refresh:
+        _CATALOG_MEM_CACHE = None
+        return None, meta
+
+    for source, cache in (("memory", _CATALOG_MEM_CACHE), ("disk", load_disk_cache() if not _CATALOG_MEM_CACHE else None)):
+        if not cache:
+            continue
+        age = now - float(cache.get("generated_at_ts", 0.0))
+        if cache.get("signature") != signature:
+            continue
+        if cache.get("version") != CATALOG_CACHE_VERSION:
+            continue
+        data = cache.get("data") or {}
+        if _catalog_has_empty_bbox(data):
+            continue
+        if age > CATALOG_CACHE_TTL_SEC:
+            continue
+        if source == "disk":
+            _CATALOG_MEM_CACHE = cache
+        meta = {
+            "cached": True,
+            "cache_source": source,
+            "cache_age_sec": round(age, 1),
+            "generated_at_ts": cache.get("generated_at_ts"),
+            "build_source": cache.get("build_source", "scan"),
+        }
+        return cache.get("data", {}), meta
+
+    return None, meta
+
+
+def store_catalog_cache(signature: dict[str, Any], data: dict[str, Any], *, build_source: str = "scan") -> dict[str, Any]:
+    global _CATALOG_MEM_CACHE
+    now = time.time()
+    payload = {
+        "version": CATALOG_CACHE_VERSION,
+        "generated_at_ts": now,
+        "signature": signature,
+        "build_source": build_source,
+        "data": data,
+    }
+    _CATALOG_MEM_CACHE = payload
+    save_disk_cache(payload)
+    return payload
+
+
+def load_dms_reports() -> tuple[dict[tuple[str, str], dict[str, int]], dict[str, dict[str, int]]] | None:
+    """Parse precomputed CSV reports: (task, pack) -> splits, task -> class_counts."""
+    if not CATALOG_USE_REPORTS or not DMS_SUMMARY_CSV.is_file():
+        return None
+
+    splits: dict[tuple[str, str], dict[str, int]] = {}
+    try:
+        with DMS_SUMMARY_CSV.open(encoding="utf-8") as f:
+            for row in csv.DictReader(f):
+                task = row.get("任务", "").strip()
+                pack = row.get("数据包", "").strip() or "default"
+                if not task:
+                    continue
+                splits[(task, pack)] = {
+                    "train": int(row.get("训练集图片") or 0),
+                    "val": int(row.get("验证集图片") or 0),
+                    "test": int(row.get("测试集图片") or 0),
+                    "total": int(row.get("图片总数") or 0),
+                }
+    except (OSError, ValueError, csv.Error):
+        return None
+
+    class_by_task: dict[str, dict[str, int]] = {}
+    if DMS_CLASS_CSV.is_file():
+        try:
+            with DMS_CLASS_CSV.open(encoding="utf-8") as f:
+                for row in csv.DictReader(f):
+                    task = row.get("任务", "").strip()
+                    cls_name = row.get("类别名", "").strip()
+                    if not task or not cls_name:
+                        continue
+                    try:
+                        cnt = int(row.get("含该类别图片数") or 0)
+                    except ValueError:
+                        continue
+                    class_by_task.setdefault(task, {})[cls_name] = cnt
+        except (OSError, ValueError, csv.Error):
+            pass
+
+    if not splits:
+        return None
+    return splits, class_by_task

platform/as_platform/data/core.py

@@ -0,0 +1,826 @@
+"""平台共享逻辑：pending、catalog、register-batch。"""
+from __future__ import annotations
+
+import json
+import math
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from as_platform.config import WORKSPACE, LANE_DATA_VIZ_ENABLED
+from as_platform.data.batch import META_FILENAME, enrich_batch, write_meta
+from as_platform.data.catalog_cache import (
+    build_catalog_signature,
+    get_cached_catalog,
+    invalidate_catalog_cache,
+    load_dms_reports,
+    store_catalog_cache,
+)
+
+MAX_LABEL_FILES_PER_PACK = 2000
+MAX_BBOX_POINTS_PER_PACK = 1500
+MAX_LANE_MASK_SAMPLES_PER_PACK = 500
+LANE_Y_BINS = 12
+IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp"}
+
+
+def load_wf() -> dict:
+    return yaml.safe_load((WORKSPACE / "workflow.registry.yaml").read_text(encoding="utf-8"))
+
+
+def proj_root(wf: dict, name: str) -> Path:
+    return (WORKSPACE / wf["projects"][name]["root"]).resolve()
+
+
+def load_pack_registry(project: str, root: Path, wf: dict) -> dict:
+    pcfg = wf["projects"][project]
+    reg_file = root / pcfg.get("packs_registry", "datasets_registry.json")
+    if reg_file.suffix in (".yaml", ".yml"):
+        return yaml.safe_load(reg_file.read_text(encoding="utf-8"))
+    return json.loads(reg_file.read_text(encoding="utf-8"))
+
+
+def _pack_registry_path(project: str, root: Path, wf: dict) -> Path:
+    pcfg = wf["projects"][project]
+    return root / pcfg.get("packs_registry", "datasets_registry.json")
+
+
+def resolve_pack(project: str, root: Path, wf: dict, name: str) -> str:
+    reg = load_pack_registry(project, root, wf)
+    name = reg.get("aliases", {}).get(name, name)
+    for p in reg.get("packs", []):
+        if p.get("name") == name:
+            return p.get("path", name)
+    if (root / name).is_dir():
+        return name
+    known = [p.get("name") for p in reg.get("packs", [])]
+    raise ValueError(f"[{project}] 未知包: {name}，已登记: {known}")
+
+
+def resolve_pack_dir(project: str, root: Path, wf: dict, name: str) -> Path:
+    return (root / resolve_pack(project, root, wf, name)).resolve()
+
+
+def _read_jsonl_tail(path: Path, n: int = 10) -> list[dict]:
+    if not path.is_file():
+        return []
+    lines = path.read_text(encoding="utf-8").strip().splitlines()
+    out = []
+    for line in lines[-n:]:
+        try:
+            out.append(json.loads(line))
+        except json.JSONDecodeError:
+            pass
+    return out
+
+
+def get_pending_report(wf: dict | None = None) -> dict[str, Any]:
+    wf = wf or load_wf()
+    report: dict[str, Any] = {
+        "workspace": str(WORKSPACE),
+        "updated_at": datetime.now(timezone.utc).isoformat(),
+        "projects": {},
+        "batches": [],
+    }
+
+    for pname, pcfg in wf["projects"].items():
+        root = proj_root(wf, pname)
+        active = set(pcfg.get("active_packs", []))
+        try:
+            reg_all = load_pack_registry(pname, root, wf)
+        except (FileNotFoundError, json.JSONDecodeError):
+            reg_all = {"packs": []}
+        all_names = {p["name"] for p in reg_all.get("packs", [])}
+        not_active = sorted(all_names - active)
+
+        proj: dict[str, Any] = {
+            "root": str(root),
+            "active_packs": list(active),
+            "not_enabled": not_active,
+            "tasks": {},
+            "task_defs": {},
+        }
+
+        if pname == "dms":
+            reg_path = root / pcfg["registry"]
+            reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
+            src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
+            ingest_log = root / "manifests" / "ingest_log.jsonl"
+            proj["recent_ingest"] = _read_jsonl_tail(ingest_log, 10)
+
+            for task, tcfg in reg.get("tasks", {}).items():
+                proj["task_defs"][task] = {
+                    "type": tcfg.get("type"),
+                    "nc": tcfg.get("nc"),
+                    "names": tcfg.get("names"),
+                    "task_dir": tcfg.get("task_dir", task),
+                }
+                inbox_batches: list[str] = []
+                ib = root / "inbox" / task
+                if ib.is_dir():
+                    inbox_batches = [
+                        d.name for d in ib.iterdir()
+                        if d.is_dir() and not d.name.startswith(".")
+                    ]
+
+                sources_pending: dict[str, list[str]] = {}
+                for pack_name in all_names:
+                    try:
+                        pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
+                    except ValueError:
+                        continue
+                    src_root = pack_dir / tcfg["task_dir"] / src_sub
+                    if src_root.is_dir():
+                        batches = [
+                            d.name for d in src_root.iterdir()
+                            if d.is_dir()
+                            and d.name not in ("_ingested", "_merged")
+                            and not d.name.startswith(".")
+                        ]
+                        if batches:
+                            sources_pending[pack_name] = batches
+
+                proj["tasks"][task] = {
+                    "inbox": inbox_batches,
+                    "sources": sources_pending,
+                }
+
+                for batch_name in inbox_batches:
+                    batch_dir = ib / batch_name
+                    report["batches"].append(
+                        enrich_batch(
+                            batch_dir,
+                            project="dms",
+                            task=task,
+                            pack=None,
+                            batch=batch_name,
+                            location="inbox",
+                        )
+                    )
+
+                for pack_name, batch_list in sources_pending.items():
+                    try:
+                        pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
+                    except ValueError:
+                        continue
+                    src_root = pack_dir / tcfg["task_dir"] / src_sub
+                    for batch_name in batch_list:
+                        batch_dir = src_root / batch_name
+                        report["batches"].append(
+                            enrich_batch(
+                                batch_dir,
+                                project="dms",
+                                task=task,
+                                pack=pack_name,
+                                batch=batch_name,
+                                location="sources",
+                            )
+                        )
+
+        if pname == "lane":
+            proj["packs"] = {}
+            for pack_name in all_names:
+                try:
+                    path = resolve_pack("lane", root, wf, pack_name)
+                except ValueError:
+                    continue
+                pack_path = root / path
+                train_lines = 0
+                tg = pack_path / "list" / "train_gt.txt"
+                if tg.is_file():
+                    train_lines = sum(1 for _ in tg.open(encoding="utf-8"))
+                proj["packs"][pack_name] = {
+                    "path": path,
+                    "train_lines": train_lines,
+                    "enabled": pack_name in active,
+                }
+                if pack_name in not_active and pack_path.is_dir():
+                    report["batches"].append(
+                        enrich_batch(
+                            pack_path,
+                            project="lane",
+                            task=None,
+                            pack=pack_name,
+                            batch=path,
+                            location="pack",
+                        )
+                    )
+
+            for child in sorted(root.iterdir()) if root.is_dir() else []:
+                if not child.is_dir() or child.name.startswith("."):
+                    continue
+                if child.name in ("lists_merged", "scripts", "inbox"):
+                    continue
+                if not child.name.startswith("DATASET-AddBy-"):
+                    continue
+                if any(
+                    p.get("path") == child.name or p.get("name") == child.name
+                    for p in reg_all.get("packs", [])
+                ):
+                    continue
+                report["batches"].append(
+                    enrich_batch(
+                        child,
+                        project="lane",
+                        task=None,
+                        pack=None,
+                        batch=child.name,
+                        location="unregistered",
+                    )
+                )
+
+            inbox_lane = root / "inbox"
+            if inbox_lane.is_dir():
+                for batch_dir in sorted(inbox_lane.iterdir()):
+                    if batch_dir.is_dir() and not batch_dir.name.startswith("."):
+                        report["batches"].append(
+                            enrich_batch(
+                                batch_dir,
+                                project="lane",
+                                task=None,
+                                pack=None,
+                                batch=batch_dir.name,
+                                location="inbox",
+                            )
+                        )
+
+        report["projects"][pname] = proj
+
+    return report
+
+
+def _parse_class_summary(text: str) -> dict[str, dict[str, int]]:
+    """解析 dataset_class_summary.txt 按 task 的类统计。"""
+    by_task: dict[str, dict[str, int]] = {}
+    current_task: str | None = None
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        if line.endswith(":") and " " not in line.rstrip(":"):
+            current_task = line.rstrip(":")
+            by_task.setdefault(current_task, {})
+            continue
+        if current_task and ":" in line:
+            parts = line.split(":", 1)
+            cls_name = parts[0].strip()
+            try:
+                count = int(re.search(r"\d+", parts[1]).group())  # type: ignore
+                by_task[current_task][cls_name] = count
+            except (AttributeError, ValueError):
+                pass
+    return by_task
+
+
+def _class_name_map(tcfg: dict[str, Any]) -> dict[int, str]:
+    names = tcfg.get("names")
+    if isinstance(names, list):
+        return {idx: str(name) for idx, name in enumerate(names)}
+    if isinstance(names, dict):
+        out: dict[int, str] = {}
+        for k, v in names.items():
+            try:
+                out[int(k)] = str(v)
+            except (TypeError, ValueError):
+                continue
+        return out
+    return {}
+
+
+def _count_images_in_dir(img_dir: Path) -> int:
+    if not img_dir.is_dir():
+        return 0
+    total = 0
+    try:
+        with os.scandir(img_dir) as it:
+            for entry in it:
+                if not entry.is_file(follow_symlinks=False):
+                    continue
+                if Path(entry.name).suffix.lower() in IMAGE_EXTS:
+                    total += 1
+    except OSError:
+        return 0
+    return total
+
+
+def _count_split_images(task_data: Path) -> dict[str, int]:
+    counts = {
+        "train": _count_images_in_dir(task_data / "images" / "train"),
+        "val": _count_images_in_dir(task_data / "images" / "val"),
+        "test": _count_images_in_dir(task_data / "images" / "test"),
+    }
+    if sum(counts.values()) == 0:
+        flat = _count_images_in_dir(task_data / "images")
+        if flat:
+            counts["train"] = flat
+    return counts
+
+
+def _iter_label_files(label_dirs: list[Path]):
+    for label_dir in label_dirs:
+        if not label_dir.is_dir():
+            continue
+        try:
+            with os.scandir(label_dir) as it:
+                stack = [entry.path for entry in it if entry.is_dir(follow_symlinks=False)]
+                files = [entry.path for entry in it if entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt")]
+        except OSError:
+            continue
+        for fp in files:
+            yield Path(fp)
+        while stack:
+            current = stack.pop()
+            try:
+                with os.scandir(current) as it:
+                    for entry in it:
+                        if entry.is_dir(follow_symlinks=False):
+                            stack.append(entry.path)
+                        elif entry.is_file(follow_symlinks=False) and entry.name.endswith(".txt"):
+                            yield Path(entry.path)
+            except OSError:
+                continue
+
+
+def _label_dirs_for_task(task_data: Path) -> list[Path]:
+    return [task_data / "labels" / "train", task_data / "labels" / "val", task_data / "labels"]
+
+
+def _parse_bbox_wh(parts: list[str]) -> list[float] | None:
+    if len(parts) < 5:
+        return None
+    try:
+        w = float(parts[3])
+        h = float(parts[4])
+        if 0.0 < w <= 1.0 and 0.0 < h <= 1.0:
+            return [round(w, 6), round(h, 6)]
+    except ValueError:
+        return None
+    return None
+
+
+def _collect_bbox_points_sample(task_data: Path, *, max_points: int = MAX_BBOX_POINTS_PER_PACK) -> list[list[float]]:
+    """Lightweight sample for scatter plot; does not scan images."""
+    bbox_points: list[list[float]] = []
+    remaining_files = MAX_LABEL_FILES_PER_PACK
+    for txt in _iter_label_files(_label_dirs_for_task(task_data)):
+        if len(bbox_points) >= max_points or remaining_files <= 0:
+            break
+        remaining_files -= 1
+        try:
+            for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
+                if len(bbox_points) >= max_points:
+                    break
+                line = line.strip()
+                if not line:
+                    continue
+                wh = _parse_bbox_wh(line.split())
+                if wh:
+                    bbox_points.append(wh)
+        except OSError:
+            continue
+    return bbox_points
+
+
+def _collect_pack_label_distribution(task_data: Path, tcfg: dict[str, Any]) -> dict[str, Any]:
+    label_dirs = _label_dirs_for_task(task_data)
+    class_counts: dict[int, int] = {}
+    bbox_points: list[list[float]] = []
+    parsed_files = 0
+    sampled = False
+    remaining = MAX_LABEL_FILES_PER_PACK
+    for txt in _iter_label_files(label_dirs):
+        if remaining <= 0:
+            sampled = True
+            break
+        remaining -= 1
+        parsed_files += 1
+        try:
+            for line in txt.read_text(encoding="utf-8", errors="ignore").splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                cls_token = line.split(maxsplit=1)[0]
+                cls_id = int(float(cls_token))
+                class_counts[cls_id] = class_counts.get(cls_id, 0) + 1
+                parts = line.split()
+                if len(bbox_points) < MAX_BBOX_POINTS_PER_PACK:
+                    wh = _parse_bbox_wh(parts)
+                    if wh:
+                        bbox_points.append(wh)
+        except OSError:
+            continue
+
+    name_map = _class_name_map(tcfg)
+    by_name: dict[str, int] = {}
+    for cls_id, cnt in sorted(class_counts.items(), key=lambda x: x[1], reverse=True):
+        key = name_map.get(cls_id, f"class_{cls_id}")
+        by_name[key] = cnt
+    return {
+        "class_counts": by_name,
+        "label_files": parsed_files,
+        "sampled": sampled,
+        "total_boxes": sum(class_counts.values()),
+        "bbox_points": bbox_points,
+    }
+
+
+def _histogram(values: list[float], bins: list[float]) -> list[dict[str, float]]:
+    if len(bins) < 2:
+        return []
+    counts = [0] * (len(bins) - 1)
+    for v in values:
+        for i in range(len(bins) - 1):
+            lo, hi = bins[i], bins[i + 1]
+            if (v >= lo and v < hi) or (i == len(bins) - 2 and v >= lo and v <= hi):
+                counts[i] += 1
+                break
+    return [
+        {"left": bins[i], "right": bins[i + 1], "count": counts[i]}
+        for i in range(len(counts))
+    ]
+
+
+def _extract_lane_mask_stats(mask_path: Path) -> dict[str, Any] | None:
+    try:
+        from PIL import Image  # type: ignore
+    except ImportError:
+        return None
+
+    try:
+        img = Image.open(mask_path).convert("L")
+    except OSError:
+        return None
+
+    w, h = img.size
+    pix = img.load()
+    if pix is None:
+        return None
+
+    lane_bins: dict[int, list[dict[str, float]]] = {}
+    present_ids: set[int] = set()
+    for y in range(h):
+        by_id: dict[int, tuple[int, int]] = {}
+        for x in range(w):
+            lane_id = int(pix[x, y])
+            if lane_id <= 0:
+                continue
+            present_ids.add(lane_id)
+            if lane_id not in by_id:
+                by_id[lane_id] = (x, x)
+            else:
+                mn, mx = by_id[lane_id]
+                if x < mn:
+                    mn = x
+                if x > mx:
+                    mx = x
+                by_id[lane_id] = (mn, mx)
+        if not by_id:
+            continue
+        y_bin = min(LANE_Y_BINS - 1, int((y / max(1, h - 1)) * LANE_Y_BINS))
+        for lane_id, (mn, mx) in by_id.items():
+            bucket = lane_bins.setdefault(lane_id, [dict(min_x=1e9, max_x=-1e9, count=0) for _ in range(LANE_Y_BINS)])
+            cur = bucket[y_bin]
+            cur["min_x"] = min(cur["min_x"], float(mn))
+            cur["max_x"] = max(cur["max_x"], float(mx))
+            cur["count"] += 1
+
+    lengths: list[float] = []
+    curvatures: list[float] = []
+    for lane_id in sorted(present_ids):
+        bins = lane_bins.get(lane_id, [])
+        centers: list[tuple[float, float]] = []
+        for i, b in enumerate(bins):
+            if b["count"] <= 0:
+                continue
+            center_x = (b["min_x"] + b["max_x"]) / 2.0
+            center_y = ((i + 0.5) / LANE_Y_BINS) * h
+            centers.append((center_x, center_y))
+        if len(centers) < 2:
+            continue
+        length = 0.0
+        for i in range(1, len(centers)):
+            dx = centers[i][0] - centers[i - 1][0]
+            dy = centers[i][1] - centers[i - 1][1]
+            length += math.sqrt(dx * dx + dy * dy)
+        lengths.append(length)
+
+        if len(centers) >= 3:
+            second_diffs = []
+            xs = [c[0] for c in centers]
+            for i in range(1, len(xs) - 1):
+                second_diffs.append(abs(xs[i + 1] - 2 * xs[i] + xs[i - 1]))
+            if second_diffs:
+                curvatures.append(sum(second_diffs) / len(second_diffs))
+
+    return {
+        "lane_count": len(present_ids),
+        "lengths": lengths,
+        "curvatures": curvatures,
+    }
+
+
+def _collect_lane_quality(pack_path: Path) -> dict[str, Any]:
+    list_files = [pack_path / "list" / "train_gt.txt", pack_path / "list" / "val_gt.txt"]
+    entries: list[Path] = []
+    for lf in list_files:
+        if not lf.is_file():
+            continue
+        try:
+            for line in lf.read_text(encoding="utf-8", errors="ignore").splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                parts = line.split()
+                if len(parts) < 2:
+                    continue
+                entries.append(pack_path / parts[1])
+                if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
+                    break
+        except OSError:
+            continue
+        if len(entries) >= MAX_LANE_MASK_SAMPLES_PER_PACK:
+            break
+
+    lane_counts: list[float] = []
+    lane_lengths: list[float] = []
+    lane_curvatures: list[float] = []
+    processed = 0
+    for ann in entries:
+        s = _extract_lane_mask_stats(ann)
+        if not s:
+            continue
+        processed += 1
+        lane_counts.append(float(s["lane_count"]))
+        lane_lengths.extend(float(x) for x in s["lengths"])
+        lane_curvatures.extend(float(x) for x in s["curvatures"])
+
+    lane_count_hist: dict[str, int] = {}
+    for c in lane_counts:
+        key = str(int(c)) if c < 8 else "8+"
+        lane_count_hist[key] = lane_count_hist.get(key, 0) + 1
+
+    return {
+        "analyzed_frames": processed,
+        "lane_count_hist": lane_count_hist,
+        "length_hist": _histogram(lane_lengths, [0, 60, 120, 180, 240, 320, 420, 560, 760, 1024]),
+        "curvature_hist": _histogram(lane_curvatures, [0, 1, 2, 4, 6, 8, 12, 16, 24, 40]),
+    }
+
+
+def _catalog_signature(wf: dict) -> dict[str, Any]:
+    return build_catalog_signature(wf, proj_root)
+
+
+def _build_catalog(wf: dict, *, prefer_reports: bool = True) -> tuple[dict[str, Any], str]:
+    out: dict[str, Any] = {"workspace": str(WORKSPACE), "dms": {}, "lane": {}}
+    build_source = "scan"
+
+    reports = load_dms_reports() if prefer_reports else None
+    report_splits: dict[tuple[str, str], dict[str, int]] = {}
+    report_classes: dict[str, dict[str, int]] = {}
+    if reports:
+        report_splits, report_classes = reports
+        build_source = "reports"
+
+    root = proj_root(wf, "dms")
+    reg_path = root / wf["projects"]["dms"]["registry"]
+    if not reg_path.is_file():
+        out["dms_error"] = f"registry not found: {reg_path}"
+        if report_splits:
+            for (task, pack_name), rep in report_splits.items():
+                entry = out["dms"].setdefault(task, {
+                    "type": "unknown",
+                    "class_counts": report_classes.get(task, {}),
+                    "packs": [],
+                })
+                entry["packs"].append({
+                    "name": pack_name,
+                    "enabled": False,
+                    "train_images": rep.get("train", 0),
+                    "val_images": rep.get("val", 0),
+                    "test_images": rep.get("test", 0),
+                    "class_counts": report_classes.get(task, {}),
+                    "label_files": 0,
+                    "total_boxes": sum(report_classes.get(task, {}).values()) if task in report_classes else 0,
+                    "sampled": True,
+                    "bbox_points": [],
+                })
+        reg = {"tasks": {}}
+    else:
+        reg = yaml.safe_load(reg_path.read_text(encoding="utf-8"))
+    try:
+        packs_reg = load_pack_registry("dms", root, wf)
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        packs_reg = {"packs": []}
+    summary_path = root / "manifests" / "dataset_class_summary.txt"
+    class_by_task = {}
+    if summary_path.is_file():
+        class_by_task = _parse_class_summary(summary_path.read_text(encoding="utf-8"))
+
+    for task, tcfg in reg.get("tasks", {}).items():
+        entry: dict[str, Any] = {
+            "type": tcfg.get("type"),
+            "nc": tcfg.get("nc"),
+            "names": tcfg.get("names"),
+            "class_counts": class_by_task.get(task, {}),
+            "packs": [],
+            "drop_paths": {
+                "inbox": str((root / "inbox" / task).resolve()),
+                "sources_template": str((root / "packs" / "<pack>" / tcfg.get("task_dir", task) / "sources" / "<batch>").resolve()),
+            },
+        }
+        for p in packs_reg.get("packs", []):
+            pack_name = p["name"]
+            try:
+                pack_dir = resolve_pack_dir("dms", root, wf, pack_name)
+            except ValueError:
+                continue
+            task_data = pack_dir / tcfg.get("task_dir", task)
+            rep = report_splits.get((task, pack_name))
+            if rep:
+                split_counts = {"train": rep["train"], "val": rep["val"], "test": rep["test"]}
+                class_counts = report_classes.get(task, class_by_task.get(task, {}))
+                bbox_points = _collect_bbox_points_sample(task_data)
+                label_distribution = {
+                    "class_counts": class_counts,
+                    "label_files": 0,
+                    "sampled": True,
+                    "total_boxes": sum(class_counts.values()) if class_counts else 0,
+                    "bbox_points": bbox_points,
+                }
+            else:
+                split_counts = _count_split_images(task_data)
+                label_distribution = _collect_pack_label_distribution(task_data, tcfg)
+                if not label_distribution["class_counts"] and task in report_classes:
+                    label_distribution["class_counts"] = report_classes[task]
+            entry["packs"].append({
+                "name": pack_name,
+                "path": p.get("path"),
+                "role": p.get("role"),
+                "frozen": p.get("frozen", False),
+                "enabled": pack_name in wf["projects"]["dms"].get("active_packs", []),
+                "train_images": split_counts.get("train", 0),
+                "val_images": split_counts.get("val", 0),
+                "test_images": split_counts.get("test", 0),
+                "class_counts": label_distribution["class_counts"],
+                "label_files": label_distribution["label_files"],
+                "total_boxes": label_distribution["total_boxes"],
+                "sampled": label_distribution["sampled"],
+                "bbox_points": label_distribution["bbox_points"],
+            })
+        if not entry["class_counts"] and task in report_classes:
+            entry["class_counts"] = report_classes[task]
+        out["dms"][task] = entry
+
+    root = proj_root(wf, "lane")
+    try:
+        reg = load_pack_registry("lane", root, wf)
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        reg = {"packs": []}
+    for p in reg.get("packs", []):
+        pack_name = p["name"]
+        path = p.get("path", pack_name)
+        pack_path = root / path
+        tg = pack_path / "list" / "train_gt.txt"
+        vg = pack_path / "list" / "val_gt.txt"
+        sg = pack_path / "list" / "test_gt.txt"
+        lane_quality = _collect_lane_quality(pack_path) if LANE_DATA_VIZ_ENABLED else {}
+        out["lane"][pack_name] = {
+            "path": path,
+            "role": p.get("role"),
+            "frozen": p.get("frozen", False),
+            "enabled": pack_name in wf["projects"]["lane"].get("active_packs", []),
+            "train_lines": sum(1 for _ in tg.open(encoding="utf-8")) if tg.is_file() else 0,
+            "val_lines": sum(1 for _ in vg.open(encoding="utf-8")) if vg.is_file() else 0,
+            "test_lines": sum(1 for _ in sg.open(encoding="utf-8")) if sg.is_file() else 0,
+            "drop_path": str(pack_path.resolve()),
+            "add_template": "python as.py add lane --src <archive> --engineer <name> --date YYYYMMDD",
+            "quality": lane_quality,
+        }
+    return out, build_source
+
+
+def get_catalog(
+    wf: dict | None = None,
+    project: str | None = None,
+    task_or_pack: str | None = None,
+    *,
+    refresh: bool = False,
+) -> dict:
+    wf = wf or load_wf()
+    sig = _catalog_signature(wf)
+    full_catalog, cache_meta = get_cached_catalog(sig, refresh=refresh)
+
+    if not full_catalog:
+        full_catalog, build_source = _build_catalog(wf, prefer_reports=not refresh)
+        store_catalog_cache(sig, full_catalog, build_source=build_source)
+        cache_meta = {"cached": False, "build_source": build_source}
+
+    result: dict[str, Any]
+    if project == "dms" and task_or_pack:
+        result = {"task": task_or_pack, **(full_catalog.get("dms", {}).get(task_or_pack, {}))}
+    elif project == "lane" and task_or_pack:
+        result = {"pack": task_or_pack, **(full_catalog.get("lane", {}).get(task_or_pack, {}))}
+    elif project in ("dms", "lane"):
+        result = {"workspace": full_catalog.get("workspace", str(WORKSPACE)), project: full_catalog.get(project, {})}
+    else:
+        result = dict(full_catalog)
+
+    if not task_or_pack and project is None:
+        result["_cache"] = cache_meta
+    return result
+
+
+def warmup_catalog_cache() -> None:
+    """Background warmup for faster first page load."""
+    try:
+        invalidate_catalog_cache()
+        get_catalog(refresh=True)
+    except Exception:
+        pass
+
+
+def register_batch(
+    wf: dict | None,
+    project: str,
+    task: str | None,
+    batch: str,
+    *,
+    pack: str | None = None,
+    stage: str = "returned",
+    engineer: str | None = None,
+    location: str = "inbox",
+) -> dict[str, Any]:
+    wf = wf or load_wf()
+    root = proj_root(wf, project)
+
+    if project == "dms":
+        if not task:
+            raise ValueError("dms register-batch 需要 task")
+        reg = yaml.safe_load((root / wf["projects"]["dms"]["registry"]).read_text(encoding="utf-8"))
+        if task not in reg.get("tasks", {}):
+            raise ValueError(f"未知 task: {task}")
+        tcfg = reg["tasks"][task]
+        if location == "sources":
+            if not pack:
+                raise ValueError("sources 位置需要 --pack")
+            pack_dir = resolve_pack_dir("dms", root, wf, pack)
+            src_sub = (reg.get("ingest") or {}).get("sources_subdir", "sources")
+            batch_dir = pack_dir / tcfg["task_dir"] / src_sub / batch
+        else:
+            batch_dir = root / "inbox" / task / batch
+    else:
+        if location == "pack" and pack:
+            try:
+                path = resolve_pack("lane", root, wf, pack)
+                batch_dir = root / path
+            except ValueError:
+                batch_dir = root / pack
+        else:
+            batch_dir = root / "inbox" / batch
+
+    if not batch_dir.is_dir():
+        raise FileNotFoundError(f"批次目录不存在: {batch_dir}")
+
+    data = {
+        "schema": "huaxu-batch-v1",
+        "project": project,
+        "task": task,
+        "pack": pack,
+        "batch": batch,
+        "stage": stage,
+        "engineer": engineer,
+        "registered_at": datetime.now(timezone.utc).isoformat(),
+    }
+    if project == "dms":
+        from as_platform.data.batch import count_images, count_label_files, dms_has_images
+
+        data["format"] = "yolo"
+        data["counts"] = {
+            "images": count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train"),
+            "labels": count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train"),
+        }
+        if not data["counts"]["images"] and dms_has_images(batch_dir):
+            data["counts"]["images"] = 1
+    else:
+        data["format"] = "ufld_archive"
+        tg = batch_dir / "list" / "train_gt.txt"
+        data["counts"] = {"images": 0, "labels": sum(1 for _ in tg.open()) if tg.is_file() else 0}
+
+    meta_path = write_meta(batch_dir, data)
+    invalidate_catalog_cache()
+    return {
+        "ok": True,
+        "meta_path": str(meta_path),
+        "batch": enrich_batch(
+            batch_dir,
+            project=project,
+            task=task,
+            pack=pack,
+            batch=batch,
+            location=location,
+        ),
+    }

platform/as_platform/data/ingest/__init__.py

@@ -0,0 +1,17 @@
+from as_platform.data.ingest.base import IngestContext, IngestAdapter, NormalizedDataset
+from as_platform.data.ingest.registry import (
+    UnknownFormatError,
+    available_formats,
+    detect_adapter,
+    inspect_uploaded_dataset,
+)
+
+__all__ = [
+    "IngestContext",
+    "IngestAdapter",
+    "NormalizedDataset",
+    "UnknownFormatError",
+    "available_formats",
+    "detect_adapter",
+    "inspect_uploaded_dataset",
+]

platform/as_platform/data/ingest/base.py

@@ -0,0 +1,46 @@
+"""Data ingest adapter base abstractions."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class IngestContext:
+    project: str
+    task: str | None
+    source_path: Path
+
+
+@dataclass
+class NormalizedDataset:
+    format_id: str
+    project: str
+    task: str | None
+    source_path: str
+    split_counts: dict[str, int] = field(default_factory=dict)
+    sample_count: int = 0
+    annotation_count: int = 0
+    artifacts: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+class IngestAdapter(ABC):
+    """Adapter interface for task-specific upload formats."""
+
+    format_id: str = "unknown"
+    projects: tuple[str, ...] = ()
+
+    @abstractmethod
+    def can_handle(self, ctx: IngestContext) -> bool:
+        raise NotImplementedError
+
+    @abstractmethod
+    def inspect(self, ctx: IngestContext) -> NormalizedDataset:
+        raise NotImplementedError

platform/as_platform/data/ingest/dms_coco.py

@@ -0,0 +1,88 @@
+"""DMS COCO-format adapter."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
+
+COCO_NAMES = ("instances_train.json", "instances_val.json", "instances_test.json", "annotations.json")
+
+
+def _read_json(path: Path) -> dict[str, Any] | None:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return None
+
+
+class DmsCocoAdapter(IngestAdapter):
+    format_id = "dms_coco"
+    projects = ("dms",)
+
+    def _find_coco_files(self, root: Path) -> list[Path]:
+        files: list[Path] = []
+        for name in COCO_NAMES:
+            p = root / "annotations" / name
+            if p.is_file():
+                files.append(p)
+        for name in COCO_NAMES:
+            p = root / name
+            if p.is_file():
+                files.append(p)
+        return files
+
+    def can_handle(self, ctx: IngestContext) -> bool:
+        root = ctx.source_path
+        return len(self._find_coco_files(root)) > 0
+
+    def inspect(self, ctx: IngestContext) -> NormalizedDataset:
+        root = ctx.source_path
+        files = self._find_coco_files(root)
+        split_counts = {"train": 0, "val": 0, "test": 0}
+        ann_count = 0
+        categories: set[str] = set()
+        warnings: list[str] = []
+        for f in files:
+            data = _read_json(f)
+            if not data:
+                warnings.append(f"failed to parse {f.name}")
+                continue
+            images = data.get("images") or []
+            anns = data.get("annotations") or []
+            cats = data.get("categories") or []
+            ann_count += len(anns)
+            for c in cats:
+                name = c.get("name")
+                if isinstance(name, str):
+                    categories.add(name)
+            lower = f.name.lower()
+            if "train" in lower:
+                split_counts["train"] += len(images)
+            elif "val" in lower:
+                split_counts["val"] += len(images)
+            elif "test" in lower:
+                split_counts["test"] += len(images)
+            else:
+                split_counts["train"] += len(images)
+
+        return NormalizedDataset(
+            format_id=self.format_id,
+            project=ctx.project,
+            task=ctx.task,
+            source_path=str(root),
+            split_counts=split_counts,
+            sample_count=sum(split_counts.values()),
+            annotation_count=ann_count,
+            artifacts=[self._artifact_name(root, f) for f in files],
+            warnings=warnings,
+            extra={"categories": sorted(categories)},
+        )
+
+    @staticmethod
+    def _artifact_name(root: Path, path: Path) -> str:
+        try:
+            return str(path.relative_to(root))
+        except ValueError:
+            return path.name

platform/as_platform/data/ingest/dms_yolo.py

@@ -0,0 +1,67 @@
+"""DMS YOLO-style dataset adapter."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
+
+IMG_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".JPG", ".JPEG", ".PNG"}
+
+
+def _count_images(path: Path) -> int:
+    if not path.is_dir():
+        return 0
+    return sum(1 for p in path.rglob("*") if p.is_file() and p.suffix in IMG_EXTS)
+
+
+def _count_txt(path: Path) -> int:
+    if not path.is_dir():
+        return 0
+    return sum(1 for p in path.rglob("*.txt") if p.is_file())
+
+
+class DmsYoloAdapter(IngestAdapter):
+    format_id = "dms_yolo"
+    projects = ("dms",)
+
+    def can_handle(self, ctx: IngestContext) -> bool:
+        root = ctx.source_path
+        return (
+            (root / "images").is_dir()
+            and (root / "labels").is_dir()
+        ) or (
+            (root / "images" / "train").is_dir()
+            and (root / "labels" / "train").is_dir()
+        )
+
+    def inspect(self, ctx: IngestContext) -> NormalizedDataset:
+        root = ctx.source_path
+        train_images = _count_images(root / "images" / "train")
+        val_images = _count_images(root / "images" / "val")
+        test_images = _count_images(root / "images" / "test")
+        if train_images + val_images + test_images == 0:
+            # fallback single-folder dataset
+            train_images = _count_images(root / "images")
+        train_labels = _count_txt(root / "labels" / "train")
+        val_labels = _count_txt(root / "labels" / "val")
+        test_labels = _count_txt(root / "labels" / "test")
+        if train_labels + val_labels + test_labels == 0:
+            train_labels = _count_txt(root / "labels")
+
+        warnings: list[str] = []
+        if train_images == 0:
+            warnings.append("train split has no images")
+        if train_labels == 0:
+            warnings.append("train split has no labels")
+
+        return NormalizedDataset(
+            format_id=self.format_id,
+            project=ctx.project,
+            task=ctx.task,
+            source_path=str(root),
+            split_counts={"train": train_images, "val": val_images, "test": test_images},
+            sample_count=train_images + val_images + test_images,
+            annotation_count=train_labels + val_labels + test_labels,
+            artifacts=["images/", "labels/"],
+            warnings=warnings,
+        )

platform/as_platform/data/ingest/lane_lines.py

@@ -0,0 +1,34 @@
+"""Lane .lines.txt adapter."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
+
+
+class LaneLinesAdapter(IngestAdapter):
+    format_id = "lane_lines"
+    projects = ("lane",)
+
+    def can_handle(self, ctx: IngestContext) -> bool:
+        root = ctx.source_path
+        return any(root.rglob("*.lines.txt"))
+
+    def inspect(self, ctx: IngestContext) -> NormalizedDataset:
+        root = ctx.source_path
+        line_files = list(root.rglob("*.lines.txt"))
+        split_counts = {"train": len(line_files), "val": 0, "test": 0}
+        warnings: list[str] = []
+        if not line_files:
+            warnings.append("no *.lines.txt found")
+        return NormalizedDataset(
+            format_id=self.format_id,
+            project=ctx.project,
+            task=ctx.task,
+            source_path=str(root),
+            split_counts=split_counts,
+            sample_count=len(line_files),
+            annotation_count=len(line_files),
+            artifacts=["*.lines.txt"],
+            warnings=warnings,
+        )

platform/as_platform/data/ingest/lane_mask.py

@@ -0,0 +1,48 @@
+"""Lane mask + list txt adapter."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
+
+
+def _line_count(path: Path) -> int:
+    if not path.is_file():
+        return 0
+    try:
+        return sum(1 for _ in path.open(encoding="utf-8", errors="ignore"))
+    except OSError:
+        return 0
+
+
+class LaneMaskAdapter(IngestAdapter):
+    format_id = "lane_mask"
+    projects = ("lane",)
+
+    def can_handle(self, ctx: IngestContext) -> bool:
+        root = ctx.source_path
+        return (root / "list" / "train_gt.txt").is_file() or (root / "train_val_gt.txt").is_file()
+
+    def inspect(self, ctx: IngestContext) -> NormalizedDataset:
+        root = ctx.source_path
+        train = _line_count(root / "list" / "train_gt.txt")
+        val = _line_count(root / "list" / "val_gt.txt")
+        test = _line_count(root / "list" / "test_gt.txt")
+        if train == 0 and (root / "train_val_gt.txt").is_file():
+            train = _line_count(root / "train_val_gt.txt")
+
+        warnings: list[str] = []
+        if train == 0:
+            warnings.append("train split list is empty")
+
+        return NormalizedDataset(
+            format_id=self.format_id,
+            project=ctx.project,
+            task=ctx.task,
+            source_path=str(root),
+            split_counts={"train": train, "val": val, "test": test},
+            sample_count=train + val + test,
+            annotation_count=train + val + test,
+            artifacts=["list/train_gt.txt", "list/val_gt.txt", "list/test_gt.txt"],
+            warnings=warnings,
+        )

platform/as_platform/data/ingest/registry.py

@@ -0,0 +1,49 @@
+"""Adapter registry and auto detection for uploaded datasets."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from as_platform.data.ingest.base import IngestAdapter, IngestContext, NormalizedDataset
+from as_platform.data.ingest.dms_coco import DmsCocoAdapter
+from as_platform.data.ingest.dms_yolo import DmsYoloAdapter
+from as_platform.data.ingest.lane_lines import LaneLinesAdapter
+from as_platform.data.ingest.lane_mask import LaneMaskAdapter
+
+
+class UnknownFormatError(ValueError):
+    pass
+
+
+ADAPTERS: tuple[IngestAdapter, ...] = (
+    DmsYoloAdapter(),
+    DmsCocoAdapter(),
+    LaneMaskAdapter(),
+    LaneLinesAdapter(),
+)
+
+
+def available_formats(project: str) -> list[str]:
+    return [a.format_id for a in ADAPTERS if project in a.projects]
+
+
+def detect_adapter(ctx: IngestContext) -> IngestAdapter:
+    for adapter in ADAPTERS:
+        if ctx.project not in adapter.projects:
+            continue
+        if adapter.can_handle(ctx):
+            return adapter
+    raise UnknownFormatError(
+        f"unable to detect format for project={ctx.project}, task={ctx.task}, "
+        f"source={ctx.source_path}. supported={available_formats(ctx.project)}"
+    )
+
+
+def inspect_uploaded_dataset(project: str, task: str | None, source_path: str | Path) -> NormalizedDataset:
+    ctx = IngestContext(project=project, task=task, source_path=Path(source_path).resolve())
+    if not ctx.source_path.exists():
+        raise FileNotFoundError(f"source path not found: {ctx.source_path}")
+    adapter = detect_adapter(ctx)
+    out = adapter.inspect(ctx)
+    # Ensure adapter id is always reflected in output.
+    out.format_id = adapter.format_id
+    return out

platform/as_platform/data/lake.py

@@ -0,0 +1,179 @@
+"""Upload candidate lifecycle for data-lake ingestion."""
+from __future__ import annotations
+
+import json
+import shutil
+import tarfile
+import uuid
+import zipfile
+from datetime import datetime
+from pathlib import Path
+from typing import BinaryIO
+
+from as_platform.config import MANIFESTS
+from as_platform.data.catalog_cache import invalidate_catalog_cache
+from as_platform.data.ingest import inspect_uploaded_dataset
+from as_platform.db.engine import session_scope
+from as_platform.db.models import DatasetCandidate
+
+LAKE_ROOT = MANIFESTS / "lake"
+UPLOAD_ROOT = LAKE_ROOT / "uploads"
+STAGING_ROOT = LAKE_ROOT / "staging"
+REPORT_ROOT = LAKE_ROOT / "reports"
+
+
+def _new_candidate_id() -> str:
+    return f"cand-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
+
+
+def _candidate_dirs(candidate_id: str) -> tuple[Path, Path, Path]:
+    upload_dir = UPLOAD_ROOT / candidate_id
+    staging_dir = STAGING_ROOT / candidate_id
+    report_file = REPORT_ROOT / f"{candidate_id}.json"
+    return upload_dir, staging_dir, report_file
+
+
+def create_uploaded_candidate(
+    *,
+    project: str,
+    task: str | None,
+    original_name: str,
+    upload_size_bytes: int,
+    submitted_by_name: str | None,
+    submitted_by_user_id: int | None,
+) -> dict:
+    candidate_id = _new_candidate_id()
+    upload_dir, _, _ = _candidate_dirs(candidate_id)
+    upload_dir.mkdir(parents=True, exist_ok=True)
+    upload_path = upload_dir / original_name
+    with session_scope() as db:
+        rec = DatasetCandidate(
+            id=candidate_id,
+            project=project,
+            task=task,
+            status="uploaded",
+            source_type="upload",
+            original_name=original_name,
+            upload_path=str(upload_path),
+            upload_size_bytes=upload_size_bytes,
+            submitted_by_name=submitted_by_name,
+            submitted_by_user_id=submitted_by_user_id,
+        )
+        db.add(rec)
+        db.flush()
+        return rec.to_dict()
+
+
+def write_candidate_upload(candidate_id: str, stream: BinaryIO, chunk_size: int = 1024 * 1024) -> str:
+    with session_scope() as db:
+        rec = db.get(DatasetCandidate, candidate_id)
+        if not rec:
+            raise ValueError(f"candidate not found: {candidate_id}")
+        path = Path(rec.upload_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("wb") as f:
+            while True:
+                chunk = stream.read(chunk_size)
+                if not chunk:
+                    break
+                f.write(chunk)
+        rec.upload_size_bytes = path.stat().st_size
+        db.flush()
+        return str(path)
+
+
+def list_candidates(limit: int = 100) -> list[dict]:
+    with session_scope() as db:
+        rows = (
+            db.query(DatasetCandidate)
+            .order_by(DatasetCandidate.created_at.desc())
+            .limit(limit)
+            .all()
+        )
+        return [r.to_dict() for r in rows]
+
+
+def get_candidate(candidate_id: str) -> dict | None:
+    with session_scope() as db:
+        rec = db.get(DatasetCandidate, candidate_id)
+        return rec.to_dict() if rec else None
+
+
+def link_candidate_analysis_job(candidate_id: str, job_id: str) -> None:
+    with session_scope() as db:
+        rec = db.get(DatasetCandidate, candidate_id)
+        if not rec:
+            raise ValueError(f"candidate not found: {candidate_id}")
+        rec.analysis_job_id = job_id
+        db.flush()
+
+
+def _extract_to_staging(upload_path: Path, staging_dir: Path) -> Path:
+    if staging_dir.exists():
+        shutil.rmtree(staging_dir)
+    staging_dir.mkdir(parents=True, exist_ok=True)
+
+    name = upload_path.name.lower()
+    if name.endswith(".zip"):
+        with zipfile.ZipFile(upload_path, "r") as zf:
+            zf.extractall(staging_dir)
+    elif name.endswith(".tar") or name.endswith(".tar.gz") or name.endswith(".tgz"):
+        with tarfile.open(upload_path, "r:*") as tf:
+            tf.extractall(staging_dir)
+    else:
+        raise ValueError(f"unsupported archive format: {upload_path.name}")
+
+    subdirs = [p for p in staging_dir.iterdir() if p.is_dir()]
+    files = [p for p in staging_dir.iterdir() if p.is_file()]
+    if len(subdirs) == 1 and not files:
+        return subdirs[0]
+    return staging_dir
+
+
+def analyze_uploaded_candidate(candidate_id: str) -> dict:
+    upload_dir, staging_dir, report_file = _candidate_dirs(candidate_id)
+    with session_scope() as db:
+        rec = db.get(DatasetCandidate, candidate_id)
+        if not rec:
+            raise ValueError(f"candidate not found: {candidate_id}")
+        rec.status = "analyzing"
+        rec.error_message = None
+        db.flush()
+        project = rec.project
+        task = rec.task
+        upload_path = Path(rec.upload_path)
+
+    if not upload_path.is_file():
+        with session_scope() as db:
+            rec = db.get(DatasetCandidate, candidate_id)
+            if rec:
+                rec.status = "failed"
+                rec.error_message = f"upload file missing: {upload_path}"
+        raise FileNotFoundError(f"upload file missing: {upload_path}")
+
+    try:
+        dataset_root = _extract_to_staging(upload_path, staging_dir)
+        normalized = inspect_uploaded_dataset(project, task, dataset_root)
+        report_file.parent.mkdir(parents=True, exist_ok=True)
+        report_file.write_text(json.dumps(normalized.to_dict(), ensure_ascii=False, indent=2), encoding="utf-8")
+        with session_scope() as db:
+            rec = db.get(DatasetCandidate, candidate_id)
+            if not rec:
+                raise ValueError(f"candidate not found during finalize: {candidate_id}")
+            rec.status = "analyzed"
+            rec.analyzed_source_path = str(dataset_root)
+            rec.format_id = normalized.format_id
+            rec.set_split_counts(normalized.split_counts)
+            rec.set_quality(normalized.to_dict())
+            rec.error_message = None
+            db.flush()
+        invalidate_catalog_cache()
+        return normalized.to_dict()
+    except Exception as e:
+        with session_scope() as db:
+            rec = db.get(DatasetCandidate, candidate_id)
+            if rec:
+                rec.status = "failed"
+                rec.error_message = str(e)
+                db.flush()
+        raise

platform/as_platform/data/organize.py

@@ -0,0 +1,38 @@
+"""数据整理：校验摘要写入 batch.meta。"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from as_platform.data.batch import META_FILENAME, count_images, count_label_files, read_meta, write_meta
+
+
+def organize_batch(batch_dir: Path, *, task: str | None = None) -> dict[str, Any]:
+    """生成整理报告并合并进 batch.meta.yaml。"""
+    batch_dir = batch_dir.resolve()
+    if not batch_dir.is_dir():
+        raise FileNotFoundError(batch_dir)
+
+    images = count_images(batch_dir / "images") + count_images(batch_dir / "images" / "train")
+    labels = count_label_files(batch_dir / "labels") + count_label_files(batch_dir / "labels" / "train")
+
+    report: dict[str, Any] = {
+        "task": task,
+        "images": images,
+        "labels": labels,
+        "pair_ratio": round(labels / images, 3) if images else 0,
+        "ready_for_ingest": images > 0 and labels > 0,
+        "issues": [],
+    }
+    if images and not labels:
+        report["issues"].append("missing_labels")
+    if labels and not images:
+        report["issues"].append("missing_images")
+
+    meta = read_meta(batch_dir) or {}
+    meta["organize_report"] = report
+    meta.setdefault("counts", {})
+    meta["counts"]["images"] = images
+    meta["counts"]["labels"] = labels
+    write_meta(batch_dir, meta)
+    return report