From 2d450715ffed2419eeaad19ce3db490d38bfe1d1 Mon Sep 17 00:00:00 2001 From: Chengfang Lu Date: Tue, 16 Jun 2026 17:36:37 +0800 Subject: [PATCH] =?UTF-8?q?docs:=20=E9=83=A8=E7=BD=B2=E4=BA=A4=E6=8E=A5?= =?UTF-8?q?=E6=96=87=E6=A1=A3=E3=80=81lake=5Fexample=20=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=A4=8D=E5=85=A5=E4=B8=8E=20CLAUDE=20=E5=BF=AB=E9=80=9F?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E6=8C=87=E5=8D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 seed_lake_example.sh 并在 init_after_clone 中调用;更新 README/CLAUDE/DEPLOY 使接手人 clone 后即可走通送标样例流程。 Co-authored-by: Cursor --- CLAUDE.md | 81 +++++++++++++++++++-------- README.md | 90 ++++++++++++++++++++++++++++-- docs/DEPLOY.md | 21 +++++++ lake/lake_example/README.md | 2 + lake/lake_example/copy_to_inbox.sh | 25 +-------- scripts/init_after_clone.sh | 14 ++++- scripts/seed_lake_example.sh | 53 ++++++++++++++++++ 7 files changed, 236 insertions(+), 50 deletions(-) create mode 100755 scripts/seed_lake_example.sh diff --git a/CLAUDE.md b/CLAUDE.md index 743cd0b..8383b21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,33 @@ This file provides guidance to Claude Code when working with code in this repository. +## Quick Deploy (Handover to Another Machine) + +```bash +git clone https://git.sanyele.com/ChengFang.LU/HSAP.git +cd HSAP +bash scripts/init_after_clone.sh # .env, feishu.env, build web, seed lake_example +bash scripts/dev_up.sh # or: make up +``` + +| Script | Purpose | +|--------|---------| +| `init_after_clone.sh` | First-time setup; calls `seed_lake_example.sh` | +| `seed_lake_example.sh` | Copy `lake/lake_example/` → data lake inbox (idempotent) | +| `dev_up.sh` | `docker compose up -d --build` (platform + CVAT + DB) | +| `build_web.sh` | `platform/web` → `platform/ui-hsap/dist` | +| `reset_labeling.sh` | Clear labeling DB; re-scan inbox | + +**Handover UI flow:** http://127.0.0.1:8787 → dev login (`AS_DEV_AUTH=true`) → 批次台账 → 扫描数据湖 → 登记 → 送标工作台 → 开标 → 我的标注. + +**Sample batches:** `lake/lake_example/datasets/manifest.yaml` (e.g. `adas/det_7cls/20260616_adas2d_pilot`). + +**Git push:** `https://git.sanyele.com/ChengFang.LU/HSAP.git` — use Gitea **Access Token** as HTTPS password if SSH fails. + +**Deploy docs:** `README.md`, `docs/DEPLOY.md`. + +--- + ## Project Overview **HSAP** (Huaxu Sentinel Active Safety Platform / 华胥 Sentinel 主动安全平台) is a truck active safety algorithm iteration platform covering DMS (Driver Monitoring), Lane detection, and ADAS perception tasks. @@ -17,14 +44,14 @@ HSAP/ ├── platform/ # Orchestration layer (API, auth, audit, jobs, web UI) ├── algorithms/ # Algorithm code layer (YOLO, UFLD adapters + registries) ├── datasets/ # Data layer (packs, inbox, sources, labeling configs) -├── scripts/ # Operational scripts (init, smoke tests, sync, worker) -├── docs/ # Documentation (20+ md files covering all aspects) -├── lake/ # Data lake staging area -├── reports/ # Reports, CSVs, figures -├── manifests/ # Runtime configs (feishu.env, DB, job logs, catalog cache) +├── lake/lake_example/ # Sample inbox batches (seeded by init_after_clone) +├── vendor/cvat/patches/ # CVAT no_auth + iframe patches +├── scripts/ # init_after_clone, dev_up, seed_lake_example, build_web +├── manifests/ # Runtime configs (feishu.env — not in git) +├── docs/ # DEPLOY.md, HANDOVER.md, ... ├── as.py # CLI entry point for workflow commands ├── workflow.registry.yaml # Central registry: projects, packs, automation rules -├── docker-compose.yml # PostgreSQL + Redis + platform + worker + optional minio +├── docker-compose.yml # platform + worker + CVAT + postgres + redis (single file) ├── Dockerfile # Python 3.11-slim, FastAPI on port 8787 └── Makefile # up/down/dev/logs/build/ps/health shortcuts ``` @@ -99,36 +126,41 @@ Central configuration for: - **Platform settings**: batch metadata schema, drop zones for inbox/sources, training tracks, agent graphs - **Automation rules**: eval-before-promote requirement, minimum delta thresholds, baseline metrics -## Docker Services +## Docker Services (single `docker-compose.yml`) | Service | Port | Description | |---------|------|-------------| | platform | 8787 | FastAPI + static web UI | -| postgres | 5432 (host mapped to 5433) | PostgreSQL 16 | -| redis | 6379 (host mapped to 6380) | Redis 7 | -| worker | - | Async job executor (same image, different command) | -| minio | 9000/9001 | Optional S3-compatible staging (profile: minio) | +| worker | — | Async job executor | +| postgres | 5433 (host) | PostgreSQL 16 | +| redis | 6380 (host) | Redis 7 | +| cvat_traefik | 8080 | CVAT gateway (UI + API) | +| cvat_server, cvat_ui, cvat_db, workers | internal | CVAT labeling engine | +| minio | 9000/9001 | Optional (profile: minio) | ## Build & Run Commands ```bash -# Quick start (Docker) -bash scripts/init_after_clone.sh # Generate .env / feishu.env +# Deploy (recommended) +bash scripts/init_after_clone.sh bash scripts/dev_up.sh # Or: make up +# Re-copy sample inbox batches +bash scripts/seed_lake_example.sh + # Local dev (no Docker for platform) pip install -r requirements.txt bash scripts/run_local.sh -# Infrastructure only (Docker) + local platform -docker compose up -d postgres redis +# Infrastructure only + local platform +docker compose up -d postgres redis cvat_traefik cvat_server bash scripts/run_local.sh +# Frontend rebuild +bash scripts/build_web.sh && docker compose restart platform + # Utilities -make logs # platform + worker logs -make down # stop all -make dev # with Vite hot reload on :5173 -make health # check API health +make logs / make down / make health ``` ## Key Environment Variables @@ -141,7 +173,10 @@ make health # check API health | `AS_JOB_EXECUTOR` | `thread` | `thread` or `worker` | | `AS_DEV_AUTH` | `false` | Bypass Feishu auth in dev | | `AS_JWT_SECRET` | - | JWT signing secret | -| `AS_WORKSPACE_ROOT` | - | External workspace path for large files | +| `AS_WORKSPACE_ROOT` | optional | External workspace (DMS/Lane large files) | +| `AS_DATA_LAKE_HOST` | `../data` | Data lake root; `送标/adas/inbox` | +| `AS_FRONTEND_URL` | `http://127.0.0.1:8787` | Browser URL (change for LAN deploy) | +| `CVAT_PUBLIC_URL` | `http://127.0.0.1:8080` | CVAT iframe URL | | `AS_FLEET_MAP_ENABLED` | `1` | Enable fleet map APIs | | `AS_FLEET_MOCK_SEED` | `1` | Seed demo vehicles on first start | @@ -150,12 +185,14 @@ make health # check API health 1. **Never commit**: `.env`, `feishu.env`, `node_modules`, `*.pt` (model weights), images/videos 2. **Python path**: Platform code runs with `PYTHONPATH=platform` from repo root 3. **Package name**: The Python package is `as_platform` (historical name preserved) -4. **Web UI**: React/Vite frontend sources are external (in workspace), built via `scripts/build_hsap_ls_ui.sh` -5. **Large files**: Images, videos, model weights live in external workspace, mounted at `/data/workspace` in Docker +4. **Web UI**: `platform/web/` built via `scripts/build_web.sh` → `platform/ui-hsap/dist/` +5. **Large files**: Training weights / bulk images in `AS_WORKSPACE_ROOT` or data lake; sample batches in `lake/lake_example/` +6. **Git**: Remote `git.sanyele.com`; HTTPS + Access Token if SSH unavailable. Company Gerrit (`10.0.0.2:29418`) is unrelated to HSAP. ## Documentation Index Key docs in `docs/`: +- `DEPLOY.md` — **New machine deploy + lake_example handover** - `DEVELOPMENT_GUIDE.md` — Architecture, conventions, design decisions - `DEVELOPMENT_ROADMAP.md` — Q2 2026 roadmap with phased milestones - `DATA_LAKE_CHECKLIST.md` — Data lake operations checklist diff --git a/README.md b/README.md index bd1813d..2c6751c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,51 @@ 克隆本仓库即可运行;**算法源码与数据集脚手架已内嵌**。大文件(图像、训练权重)通过外部 `workspace` 或 **送标数据湖** 挂载,不进入 Git。 -**新机器部署请直接看 → [docs/DEPLOY.md](docs/DEPLOY.md)** +**新机器 3 步部署 → 见下方 [快速部署到其他机器](#快速部署到其他机器)** · 详细文档 [docs/DEPLOY.md](docs/DEPLOY.md) + +--- + +## 快速部署到其他机器 + +在目标 Linux 机器上执行(需 Docker + Node 18+): + +```bash +# 1. 克隆(HTTPS 需 Gitea 用户名 + Access Token 作密码;或配置 SSH 公钥) +git clone https://git.sanyele.com/ChengFang.LU/HSAP.git +cd HSAP + +# 2. 初始化:.env、feishu.env、前端构建、lake_example 样例植入 +bash scripts/init_after_clone.sh + +# 3. 启动平台 + CVAT +bash scripts/dev_up.sh +``` + +浏览器打开 http://127.0.0.1:8787 → 开发登录 → **批次台账 → 扫描数据湖 → 登记 → 送标工作台开标**(样例已自动就位)。 + +| 检查项 | 命令 / 地址 | +|--------|-------------| +| 平台健康 | `make health` | +| 容器状态 | `docker compose ps` | +| 样例批次 | `lake/lake_example/datasets/manifest.yaml` | +| 局域网访问 | 改 `.env` 中 `AS_FRONTEND_URL`、`CVAT_PUBLIC_URL` 为 `http://:8787` / `:8080` | + +**目录布局(推荐与仓库平级):** + +```text +/opt/DATA/ +├── HSAP/ ← git clone +└── data/ ← AS_DATA_LAKE_HOST,init 自动创建 送标/adas/inbox +``` + +`AS_WORKSPACE_ROOT`(DMS 训练大图)**验收送标流程不必配置**;有外部 workspace 时再在 `.env` 填写。 + +**推送代码到远端:** + +```bash +git push origin main +# 用户名 Chengfang.LU,密码填 Gitea Access Token(设置 → 应用 → 生成令牌) +``` --- @@ -55,12 +99,11 @@ HSAP/ git clone https://git.sanyele.com/ChengFang.LU/HSAP.git cd HSAP -bash scripts/init_after_clone.sh # 生成 .env / feishu.env,构建前端 -# 编辑 .env:AS_WORKSPACE_ROOT、AS_DATA_LAKE_HOST、AS_FRONTEND_URL +bash scripts/init_after_clone.sh # .env / feishu.env、构建前端、植入 lake_example 样例 bash scripts/dev_up.sh # 或: make up ``` -`make up` / `dev_up.sh` 一次启动 **HSAP 平台 + 内置 CVAT**(无需单独部署 CVAT,无需 CVAT 账号登录)。 +按上面两步即可 **启动平台**;`init_after_clone.sh` 会自动把 [`lake/lake_example`](lake/lake_example/) 里的示例送标批次复制到数据湖 inbox,接手人无需再手动拷文件。 | 服务 | 地址 | 说明 | |------|------|------| @@ -79,7 +122,42 @@ make health # API 健康检查 **改前端后:** 须 `bash scripts/build_web.sh` 并 `docker compose restart platform`。 -**部署到他人电脑 / 局域网:** 修改 `.env` 中 `AS_FRONTEND_URL`、`CVAT_PUBLIC_URL`、`FEISHU_REDIRECT_URI` 为实际访问地址,详见 [docs/DEPLOY.md](docs/DEPLOY.md)。 +**局域网部署:** 修改 `.env` 中 `AS_FRONTEND_URL`、`CVAT_PUBLIC_URL`,详见 [docs/DEPLOY.md](docs/DEPLOY.md)。 + +--- + +## 接手走通送标流程(lake_example) + +仓库内 [`lake/lake_example/`](lake/lake_example/) 含 **DMS / ADAS 2D / ADAS 3D / Lane** 各一条样例批次(含少量图片),用于验收全流程。 + +### 一键植入(部署时自动执行) + +```bash +bash scripts/init_after_clone.sh # 已包含 seed +# 或单独执行: +bash scripts/seed_lake_example.sh +``` + +样例落盘位置: + +| 业务 | 路径 | 样例批次 | +|------|------|----------| +| ADAS 2D | `data/送标/adas/inbox/det_7cls/` | `20260616_adas2d_pilot` | +| ADAS 3D | `data/送标/adas/inbox/cuboid_7cls/` | `20260616_3d_pilot` | +| DMS | `datasets/dms/inbox/addw/` 等 | `20260616_addw_pilot` 等 | +| Lane | `datasets/lane/inbox/` | `20260616_lane_pilot` | + +默认 `AS_DATA_LAKE_HOST=../data`(与 HSAP 仓库平级)。完整清单见 [`lake/lake_example/datasets/manifest.yaml`](lake/lake_example/datasets/manifest.yaml)。 + +### UI 操作顺序 + +1. 打开 http://127.0.0.1:8787 → 开发登录 +2. **批次台账** → **扫描数据湖** → 勾选样例批次 → **登记到台账** +3. **送标工作台** → 选批次(建议先试 `adas / det_7cls / 20260616_adas2d_pilot`)→ **开标** +4. **标注进度** → **分配任务** → 标注员进入 **我的标注** → CVAT 画框保存 +5. 协调员 **提交批次** → 审核员 **审核队列** 通过 → 导出入库 + +清空标注记录重来:`bash scripts/reset_labeling.sh`(不删 inbox 文件,可再次扫描登记)。 --- @@ -127,6 +205,8 @@ ADAS 7 类:`car`, `pedestrian`, `truck`, `bus`, `motorcycle`, `tricycle`, `tra 批次目录示例见 `lake/lake_example/`。登记:**批次台账 → 扫描数据湖 → 登记**。 +部署时执行 `bash scripts/seed_lake_example.sh`(已包含在 `init_after_clone.sh`)可自动植入样例批次。 + 清空送标/标注 DB(保留账号):`bash scripts/reset_labeling.sh` --- diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 2231429..21a5d4e 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -38,6 +38,27 @@ bash scripts/init_after_clone.sh - 从 `manifests/feishu.env.example` 生成 `manifests/feishu.env` - 若上级目录存在 `workspace/`、`data/`,自动写入挂载路径 - 首次构建前端 → `platform/ui-hsap/dist/` +- **植入 `lake/lake_example` 样例送标批次** → 数据湖 inbox(接手人可直接走流程) + +--- + +## 3.5 样例送标数据(lake_example) + +`init_after_clone.sh` 已自动执行 `scripts/seed_lake_example.sh`。也可单独重跑: + +```bash +bash scripts/seed_lake_example.sh +``` + +推荐验收批次: + +| 类型 | 批次名 | +|------|--------| +| ADAS 2D | `20260616_adas2d_pilot` | +| ADAS 3D | `20260616_3d_pilot` | +| DMS | `20260616_addw_pilot` | + +平台启动后:**批次台账 → 扫描数据湖 → 登记 → 送标工作台开标**。详见 [README 接手流程](../README.md#接手走通送标流程lake_example)。 --- diff --git a/lake/lake_example/README.md b/lake/lake_example/README.md index 72e3458..1d07674 100644 --- a/lake/lake_example/README.md +++ b/lake/lake_example/README.md @@ -15,6 +15,8 @@ DMS 舱内(addw/ddaw/dam 等)仍在 `datasets/dms/inbox/`。 ## 复制到数据湖 ```bash +bash ../../scripts/seed_lake_example.sh +# 或在本目录: bash copy_to_inbox.sh ``` diff --git a/lake/lake_example/copy_to_inbox.sh b/lake/lake_example/copy_to_inbox.sh index 2c2d446..2a0b2a3 100755 --- a/lake/lake_example/copy_to_inbox.sh +++ b/lake/lake_example/copy_to_inbox.sh @@ -1,24 +1,5 @@ #!/usr/bin/env bash -# 将 lake_example 样例复制到 HSAP 数据湖 inbox(不覆盖已有批次) +# 兼容入口:请优先使用 scripts/seed_lake_example.sh set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -HSAP_ROOT="${HSAP_ROOT:-$(cd "$SCRIPT_DIR/../HSAP" && pwd)}" -SRC="$SCRIPT_DIR/datasets" - -echo "HSAP_ROOT=$HSAP_ROOT" -echo "源: $SRC" -echo "" -echo "业务线 → 目标路径:" -echo " DMS cabin (addw/ddaw/...) → datasets/dms/inbox/" -echo " ADAS 2D (adas/inbox/det_7cls) → datasets/adas/inbox/det_7cls/" -echo " ADAS 3D (adas/inbox/cuboid) → datasets/adas/inbox/cuboid_7cls/" -echo " 车道线 (lane/inbox) → datasets/lane/inbox/" -echo "" - -rsync -av --ignore-existing "$SRC/dms/inbox/" "$HSAP_ROOT/datasets/dms/inbox/" -rsync -av --ignore-existing "$SRC/adas/inbox/" "$HSAP_ROOT/datasets/adas/inbox/" -rsync -av --ignore-existing "$SRC/lane/inbox/" "$HSAP_ROOT/datasets/lane/inbox/" - -echo "" -echo "完成。请到 批次台账 → 扫描数据湖 → 登记。" -echo "索引: $SRC/README.md | manifest: $SRC/manifest.yaml" +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +exec bash "$ROOT/scripts/seed_lake_example.sh" diff --git a/scripts/init_after_clone.sh b/scripts/init_after_clone.sh index 4c58da0..fbc8726 100755 --- a/scripts/init_after_clone.sh +++ b/scripts/init_after_clone.sh @@ -9,13 +9,21 @@ cd "$ROOT" DEFAULT_WS="$(cd "$ROOT/.." && pwd)/workspace" DEFAULT_DATA="$(cd "$ROOT/.." && pwd)/data" +mkdir -p "$DEFAULT_DATA/送标/adas/inbox" if grep -q '^# AS_WORKSPACE_ROOT=' .env && [[ -d "$DEFAULT_WS/DMS" ]]; then sed -i "s|^# AS_WORKSPACE_ROOT=.*|AS_WORKSPACE_ROOT=$DEFAULT_WS|" .env echo "已设置 AS_WORKSPACE_ROOT=$DEFAULT_WS" fi -if grep -q '^# AS_DATA_LAKE_HOST=' .env && [[ -d "$DEFAULT_DATA" ]]; then +if ! grep -q '^AS_DATA_LAKE_HOST=' .env 2>/dev/null; then + if grep -q '^# AS_DATA_LAKE_HOST=' .env; then + sed -i "s|^# AS_DATA_LAKE_HOST=.*|AS_DATA_LAKE_HOST=$DEFAULT_DATA|" .env + else + echo "AS_DATA_LAKE_HOST=$DEFAULT_DATA" >> .env + fi + echo "已设置 AS_DATA_LAKE_HOST=$DEFAULT_DATA" +elif grep -q '^# AS_DATA_LAKE_HOST=' .env && [[ -d "$DEFAULT_DATA" ]]; then sed -i "s|^# AS_DATA_LAKE_HOST=.*|AS_DATA_LAKE_HOST=$DEFAULT_DATA|" .env echo "已设置 AS_DATA_LAKE_HOST=$DEFAULT_DATA" fi @@ -42,6 +50,10 @@ if [[ ! -f platform/ui-hsap/dist/index.html ]]; then fi fi +echo "" +echo "植入送标样例(lake_example)…" +bash scripts/seed_lake_example.sh + echo "" echo "下一步:" echo " 1. 编辑 .env — 路径与 AS_FRONTEND_URL / CVAT_PUBLIC_URL(局域网部署必改)" diff --git a/scripts/seed_lake_example.sh b/scripts/seed_lake_example.sh new file mode 100755 index 0000000..2ae66df --- /dev/null +++ b/scripts/seed_lake_example.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# 将 lake/lake_example 样例批次复制到数据湖 inbox,供接手人走通送标流程 +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +SRC="$ROOT/lake/lake_example/datasets" +if [[ ! -d "$SRC/dms/inbox" ]]; then + echo "未找到 $SRC,请确认已 git clone 完整仓库" + exit 1 +fi + +# 数据湖根目录(与 docker-compose AS_DATA_LAKE_HOST 一致) +DATA_LAKE="$(cd "$ROOT/.." && pwd)/data" +if [[ -f .env ]]; then + val="$(grep -E '^AS_DATA_LAKE_HOST=' .env 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"' || true)" + [[ -n "$val" ]] && DATA_LAKE="$val" +fi + +ADAS_LAKE="$DATA_LAKE/送标/adas" +mkdir -p "$ADAS_LAKE/inbox" "$ROOT/datasets/dms/inbox" "$ROOT/datasets/lane/inbox" + +# ADAS 走数据湖挂载;DMS/Lane 在仓库 datasets/ 下 +if [[ ! -e "$ROOT/datasets/adas" ]]; then + ln -sfn ../../data/送标/adas "$ROOT/datasets/adas" + echo "已创建 datasets/adas → data/送标/adas" +elif [[ -L "$ROOT/datasets/adas" ]]; then + echo "datasets/adas 软链已存在" +else + echo "警告: datasets/adas 已存在且非软链,ADAS 样例将写入该目录" +fi + +RSYNC=(rsync -rlpt --no-owner --no-group --ignore-existing) + +echo "数据湖根: $DATA_LAKE" +echo "复制 lake_example → inbox(不覆盖已有批次)…" +echo "" + +"${RSYNC[@]}" "$SRC/dms/inbox/" "$ROOT/datasets/dms/inbox/" +"${RSYNC[@]}" "$SRC/lane/inbox/" "$ROOT/datasets/lane/inbox/" +mkdir -p "$ADAS_LAKE/inbox" +"${RSYNC[@]}" "$SRC/adas/inbox/" "$ADAS_LAKE/inbox/" + +echo "" +echo "样例批次已就位。manifest: lake/lake_example/datasets/manifest.yaml" +echo "" +echo "平台启动后走通流程:" +echo " 1. 登录 http://127.0.0.1:8787 (AS_DEV_AUTH=true 开发登录)" +echo " 2. 批次台账 → 扫描数据湖 → 勾选样例 → 登记到台账" +echo " 3. 送标工作台 → 选择批次 → 开标" +echo " 4. 标注进度 → 分配任务 → 标注员「我的标注」进入 CVAT" +echo "" +echo "推荐先试: adas/det_7cls/20260616_adas2d_pilot 或 adas/cuboid_7cls/20260616_3d_pilot"