feat: initial HSAP platform
Huaxu Sentinel Active Safety Platform with embedded algorithm code, Docker Compose setup, and vendored dataset scaffolds for clone-and-run. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
100
scripts/db_migrate_from_sqlite.py
Normal file
100
scripts/db_migrate_from_sqlite.py
Normal file
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""将 SQLite 遗留库 manifests/platform.db 迁移到 PostgreSQL。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "platform"))
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import joinedload, sessionmaker
|
||||
|
||||
from as_platform.config import DATABASE_URL, IS_POSTGRES, SQLITE_LEGACY_PATH
|
||||
from as_platform.db.engine import Base
|
||||
from as_platform.db.init_db import init_database
|
||||
from as_platform.db.models import Approval, Job, Permission, Role, User
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not IS_POSTGRES:
|
||||
print("目标 AS_DATABASE_URL 须为 postgresql://…", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not SQLITE_LEGACY_PATH.is_file():
|
||||
print(f"无 SQLite 文件: {SQLITE_LEGACY_PATH},跳过")
|
||||
init_database()
|
||||
return
|
||||
|
||||
sqlite_url = f"sqlite:///{SQLITE_LEGACY_PATH}"
|
||||
src = create_engine(sqlite_url, future=True)
|
||||
dst = create_engine(DATABASE_URL, future=True)
|
||||
SrcSession = sessionmaker(bind=src)
|
||||
DstSession = sessionmaker(bind=dst)
|
||||
|
||||
Base.metadata.create_all(bind=dst)
|
||||
|
||||
with DstSession() as db:
|
||||
if db.query(User).count() > 0:
|
||||
print("PostgreSQL 已有数据,跳过迁移(可加 --force 清空后重迁)")
|
||||
if "--force" not in sys.argv:
|
||||
return
|
||||
print("清空目标库…")
|
||||
db.query(Job).delete()
|
||||
db.query(Approval).delete()
|
||||
from as_platform.db.models import role_permissions, user_roles
|
||||
db.execute(user_roles.delete())
|
||||
db.execute(role_permissions.delete())
|
||||
db.query(User).delete()
|
||||
db.query(Role).delete()
|
||||
db.query(Permission).delete()
|
||||
db.commit()
|
||||
|
||||
order = [Permission, Role, User, Approval, Job]
|
||||
with SrcSession() as sdb, DstSession() as ddb:
|
||||
perm_map = {p.id: p for p in sdb.query(Permission).all()}
|
||||
for p in perm_map.values():
|
||||
ddb.merge(Permission(id=p.id, code=p.code, name=p.name))
|
||||
ddb.flush()
|
||||
for r in sdb.query(Role).options(joinedload(Role.permissions)).all():
|
||||
nr = Role(id=r.id, code=r.code, name=r.name)
|
||||
ddb.add(nr)
|
||||
ddb.flush()
|
||||
nr.permissions = [ddb.get(Permission, p.id) for p in r.permissions]
|
||||
for u in sdb.query(User).options(joinedload(User.roles)).all():
|
||||
nu = User(
|
||||
id=u.id,
|
||||
feishu_open_id=u.feishu_open_id,
|
||||
feishu_union_id=u.feishu_union_id,
|
||||
name=u.name,
|
||||
email=u.email,
|
||||
avatar_url=u.avatar_url,
|
||||
is_active=bool(u.is_active),
|
||||
created_at=u.created_at,
|
||||
updated_at=u.updated_at,
|
||||
)
|
||||
ddb.add(nu)
|
||||
ddb.flush()
|
||||
nu.roles = [ddb.get(Role, r.id) for r in u.roles]
|
||||
for a in sdb.query(Approval).all():
|
||||
ddb.merge(Approval(
|
||||
id=a.id, status=a.status, action=a.action, action_label=a.action_label,
|
||||
params_json=a.params_json, note=a.note,
|
||||
submitted_by_user_id=a.submitted_by_user_id, submitted_by_name=a.submitted_by_name,
|
||||
submitted_at=a.submitted_at, reviewed_by_user_id=a.reviewed_by_user_id,
|
||||
reviewed_by_name=a.reviewed_by_name, reviewed_at=a.reviewed_at,
|
||||
review_comment=a.review_comment, job_id=a.job_id,
|
||||
executed_at=a.executed_at, result_json=a.result_json,
|
||||
))
|
||||
for j in sdb.query(Job).all():
|
||||
ddb.merge(Job(
|
||||
id=j.id, status=j.status, action=j.action, params_json=j.params_json,
|
||||
approval_id=j.approval_id, created_at=j.created_at,
|
||||
started_at=j.started_at, finished_at=j.finished_at, result_json=j.result_json,
|
||||
))
|
||||
ddb.commit()
|
||||
print("迁移完成 → PostgreSQL")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
28
scripts/dev_up.sh
Executable file
28
scripts/dev_up.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# 一键启动完整开发环境:postgres + redis + platform + worker
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
if [[ ! -f .env ]]; then
|
||||
cp .env.example .env
|
||||
echo "已创建 .env(默认挂载本仓库;大文件数据可设置 AS_WORKSPACE_ROOT)"
|
||||
fi
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "未安装 Docker。Ubuntu: sudo apt install docker.io docker-compose-v2"
|
||||
echo "或: https://docs.docker.com/engine/install/"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker compose up -d --build "$@"
|
||||
|
||||
echo ""
|
||||
echo "服务:"
|
||||
echo " 平台 UI+API http://127.0.0.1:${AS_PLATFORM_PORT:-8787}"
|
||||
echo " PostgreSQL localhost:${AS_DB_PORT:-5432}"
|
||||
echo " Redis localhost:${AS_REDIS_PORT:-6379}"
|
||||
echo ""
|
||||
echo "React 热更新: docker compose --profile dev up -d web-dev → :5173"
|
||||
echo "日志: docker compose logs -f platform worker"
|
||||
echo "停止: docker compose down"
|
||||
154
scripts/eval_dms.py
Executable file
154
scripts/eval_dms.py
Executable file
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""DMS 标准评估:yolo val → manifests/eval_log.jsonl,可选基线对比。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
WORKSPACE = Path(__file__).resolve().parents[1]
|
||||
DATASET = (WORKSPACE / "datasets/dms").resolve()
|
||||
WF_PATH = WORKSPACE / "workflow.registry.yaml"
|
||||
|
||||
|
||||
def latest_run_weights(yolo_root: Path, mode: str, task: str) -> Path | None:
|
||||
base = yolo_root / "runs" / mode
|
||||
if not base.is_dir():
|
||||
return None
|
||||
cands = sorted(base.glob(f"{task}_*"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
for run in cands:
|
||||
w = run / "weights" / "best.pt"
|
||||
if w.is_file():
|
||||
return w
|
||||
return None
|
||||
|
||||
|
||||
def parse_val_metrics(stdout: str) -> dict:
|
||||
metrics = {}
|
||||
for key, pat in [
|
||||
("map50", r"mAP50[:\s]+([\d.]+)"),
|
||||
("map50_95", r"mAP50-95[:\s]+([\d.]+)"),
|
||||
("precision", r"Precision[:\s]+([\d.]+)"),
|
||||
("recall", r"Recall[:\s]+([\d.]+)"),
|
||||
]:
|
||||
m = re.search(pat, stdout, re.I)
|
||||
if m:
|
||||
metrics[key] = float(m.group(1))
|
||||
return metrics
|
||||
|
||||
|
||||
def append_eval_log(record: dict) -> None:
|
||||
log = DATASET / "manifests" / "eval_log.jsonl"
|
||||
log.parent.mkdir(parents=True, exist_ok=True)
|
||||
record["ts"] = datetime.now(timezone.utc).isoformat()
|
||||
with log.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def load_train_versions() -> dict:
|
||||
p = DATASET / "manifests" / "train_versions.yaml"
|
||||
return yaml.safe_load(p.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def save_train_versions(data: dict) -> None:
|
||||
p = DATASET / "manifests" / "train_versions.yaml"
|
||||
p.write_text(yaml.dump(data, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("task")
|
||||
ap.add_argument("--weights", type=Path, help="默认 candidate → current → 最近 runs/*/best.pt")
|
||||
ap.add_argument("--save-candidate", action="store_true", help="将权重写入 train_versions candidate")
|
||||
args = ap.parse_args()
|
||||
|
||||
reg = yaml.safe_load((DATASET / "datasets.registry.yaml").read_text())
|
||||
wf = yaml.safe_load(WF_PATH.read_text())
|
||||
versions = load_train_versions()
|
||||
task = args.task
|
||||
if task not in reg["tasks"]:
|
||||
sys.exit(f"未知 task: {task}")
|
||||
|
||||
tcfg = reg["tasks"][task]
|
||||
typ = tcfg["type"]
|
||||
mode = {"detect": "detect", "pose": "pose", "classify": "classify"}[typ]
|
||||
yaml_path = DATASET / "manifests" / "yaml_active" / f"{task}.yaml"
|
||||
if not yaml_path.is_file():
|
||||
sys.exit(f"缺少 {yaml_path},请先: python as.py build dms")
|
||||
|
||||
tv = versions.get(task, {})
|
||||
weights = args.weights
|
||||
if weights is None:
|
||||
for key in ("candidate", "current"):
|
||||
w = tv.get(key)
|
||||
if w and Path(w).is_file():
|
||||
weights = Path(w)
|
||||
break
|
||||
yolo_root = WORKSPACE / "dms/code/yolo26_rknn_ultralytics-main"
|
||||
if weights is None:
|
||||
weights = latest_run_weights(yolo_root, mode, task)
|
||||
if weights is None or not Path(weights).is_file():
|
||||
sys.exit("未找到权重,请 --weights 或先 train")
|
||||
|
||||
imgsz = reg["train"][typ]["imgsz"]
|
||||
active_packs = wf["projects"]["dms"]["active_packs"]
|
||||
name = f"{task}_eval_{datetime.now().strftime('%Y%m%d_%H%M')}"
|
||||
|
||||
cmd = [
|
||||
"yolo", mode, "val",
|
||||
f"data={yaml_path}",
|
||||
f"model={weights}",
|
||||
f"imgsz={imgsz}",
|
||||
f"project=runs/{mode}",
|
||||
f"name={name}",
|
||||
]
|
||||
print(" ".join(cmd))
|
||||
proc = subprocess.run(cmd, cwd=yolo_root, capture_output=True, text=True)
|
||||
out = proc.stdout + proc.stderr
|
||||
print(out)
|
||||
if proc.returncode != 0:
|
||||
return proc.returncode
|
||||
|
||||
metrics = parse_val_metrics(out)
|
||||
baseline = (wf.get("automation") or {}).get("baseline_metrics", {}).get(task, {})
|
||||
record = {
|
||||
"task": task,
|
||||
"type": typ,
|
||||
"weights": str(Path(weights).resolve()),
|
||||
"active_packs": active_packs,
|
||||
"data_yaml": str(yaml_path),
|
||||
"metrics": metrics,
|
||||
"baseline": baseline,
|
||||
"run_name": name,
|
||||
}
|
||||
if baseline and metrics.get("map50") is not None and baseline.get("map50") is not None:
|
||||
record["delta_map50"] = metrics["map50"] - float(baseline["map50"])
|
||||
|
||||
append_eval_log(record)
|
||||
versions[task]["last_eval"] = record
|
||||
save_train_versions(versions)
|
||||
|
||||
if args.save_candidate:
|
||||
versions[task]["candidate"] = str(Path(weights).resolve())
|
||||
save_train_versions(versions)
|
||||
|
||||
print(f"\nmetrics: {metrics}")
|
||||
if record.get("delta_map50") is not None:
|
||||
print(f"delta mAP50 vs baseline: {record['delta_map50']:+.4f}")
|
||||
|
||||
auto = wf.get("automation") or {}
|
||||
min_delta = float(auto.get("min_delta_map50", -1.0))
|
||||
if record.get("delta_map50") is not None and record["delta_map50"] < min_delta:
|
||||
print(f"FAIL: delta mAP50 < {min_delta}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
20
scripts/init_after_clone.sh
Executable file
20
scripts/init_after_clone.sh
Executable file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Git clone 后首次初始化
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
[[ -f .env ]] || { cp .env.example .env && echo "已创建 .env"; }
|
||||
[[ -f manifests/feishu.env ]] || { cp manifests/feishu.env.example manifests/feishu.env && echo "已创建 manifests/feishu.env"; }
|
||||
|
||||
if [[ -n "${AS_WORKSPACE_ROOT:-}" ]] && [[ -d "${AS_WORKSPACE_ROOT}/DMS" || -d "${AS_WORKSPACE_ROOT}/LaneDection" ]]; then
|
||||
echo "检测到 AS_WORKSPACE_ROOT,切换为 workspace 软链…"
|
||||
bash scripts/setup_links.sh
|
||||
else
|
||||
echo "使用仓库内嵌 algorithms/datasets(默认)"
|
||||
echo "若有外部 workspace: export AS_WORKSPACE_ROOT=/path/to/workspace && bash scripts/setup_links.sh"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Docker 启动: bash scripts/dev_up.sh"
|
||||
echo "本机启动: bash scripts/run_local.sh"
|
||||
37
scripts/run_local.sh
Normal file
37
scripts/run_local.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# 本机直跑(不用 Docker 跑 platform/worker)
|
||||
# 依赖: pip install -r requirements.txt
|
||||
# PostgreSQL 可选;不可用时自动回退 SQLite
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
export PYTHONPATH="${ROOT}/platform${PYTHONPATH:+:$PYTHONPATH}"
|
||||
|
||||
if [[ -f "$ROOT/manifests/feishu.env" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
source "$ROOT/manifests/feishu.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
export AS_JOB_EXECUTOR="${AS_JOB_EXECUTOR:-thread}"
|
||||
|
||||
cd "$ROOT"
|
||||
bash scripts/setup_links.sh 2>/dev/null || true
|
||||
|
||||
if ! python - <<'PY' 2>/dev/null; then
|
||||
from as_platform.db.engine import check_connection
|
||||
raise SystemExit(0 if check_connection() else 1)
|
||||
PY
|
||||
export AS_DATABASE_URL="${AS_DATABASE_URL:-sqlite:///${ROOT}/manifests/platform.db}"
|
||||
echo "PostgreSQL 不可用,使用 SQLite: ${AS_DATABASE_URL}"
|
||||
fi
|
||||
|
||||
python scripts/wait_for_db.py
|
||||
python scripts/db_migrate_from_sqlite.py 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "启动平台 API: http://127.0.0.1:${AS_PLATFORM_PORT:-8787}"
|
||||
echo "开发登录: AS_DEV_AUTH=true 时在登录页点「开发登录」"
|
||||
echo ""
|
||||
|
||||
exec python -m as_platform.api.server --host "${AS_PLATFORM_HOST:-127.0.0.1}" --port "${AS_PLATFORM_PORT:-8787}" "$@"
|
||||
58
scripts/setup_links.sh
Executable file
58
scripts/setup_links.sh
Executable file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# 将 datasets/ 与 algorithms/*/code 软链到外部 workspace(可选,覆盖仓库内嵌副本)
|
||||
#
|
||||
# 适用:本地 monorepo 布局 DATA/{HSAP,workspace}
|
||||
# 克隆单仓库且无外置数据时无需运行本脚本。
|
||||
#
|
||||
# 用法:
|
||||
# export AS_WORKSPACE_ROOT=/path/to/workspace
|
||||
# bash scripts/setup_links.sh
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
WS="${AS_WORKSPACE_ROOT:-$(cd "$ROOT/.." && pwd)/workspace}"
|
||||
|
||||
if [[ ! -d "$WS/DMS" ]] && [[ ! -d "$WS/LaneDection" ]]; then
|
||||
echo "跳过: workspace 无效 ($WS),继续使用仓库内嵌 algorithms/datasets"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "HSAP: $ROOT"
|
||||
echo "workspace: $WS"
|
||||
|
||||
link_dir() {
|
||||
local target="$1" link="$2"
|
||||
if [[ -d "$link" && ! -L "$link" ]]; then
|
||||
echo " 备份内嵌目录 → ${link}.embedded.bak"
|
||||
mv "$link" "${link}.embedded.bak"
|
||||
elif [[ -L "$link" ]]; then
|
||||
rm -f "$link"
|
||||
fi
|
||||
mkdir -p "$(dirname "$link")"
|
||||
ln -sfn "$target" "$link"
|
||||
echo " $link -> $target"
|
||||
}
|
||||
|
||||
mkdir -p "$WS/Lane" "$ROOT/datasets" "$ROOT/algorithms/dms_yolo" "$ROOT/algorithms/lane_ufld"
|
||||
|
||||
# workspace 内 Lane 逻辑入口(可选)
|
||||
[[ -d "$WS/lane" ]] && ln -sfn "$WS/lane" "$WS/Lane/dataset" 2>/dev/null || true
|
||||
[[ -d "$WS/LaneDection/Code" ]] && ln -sfn "$WS/LaneDection/Code" "$WS/Lane/code" 2>/dev/null || true
|
||||
|
||||
echo ">>> datasets"
|
||||
link_dir "$WS/DMS/DATASET" "$ROOT/datasets/dms"
|
||||
link_dir "$WS/lane" "$ROOT/datasets/lane"
|
||||
|
||||
echo ">>> algorithms"
|
||||
link_dir "$WS/DMS/Code/yolo26_rknn_ultralytics-main" "$ROOT/algorithms/dms_yolo/code"
|
||||
link_dir "$WS/LaneDection/Code" "$ROOT/algorithms/lane_ufld/code"
|
||||
|
||||
cat > "$ROOT/manifests/repo_layout.json" <<EOF
|
||||
{
|
||||
"layout": "workspace_symlinks",
|
||||
"workspace": "$WS",
|
||||
"linked_at": "$(date -Iseconds)"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "完成(workspace 软链模式)"
|
||||
78
scripts/sync_to_server.sh
Executable file
78
scripts/sync_to_server.sh
Executable file
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# 同步 HSAP 工作区到训练服务器(展开软链)
|
||||
#
|
||||
# 用法:
|
||||
# ./scripts/sync_to_server.sh USER@HOST:/opt/HSAP
|
||||
# ./scripts/sync_to_server.sh USER@HOST:/opt/HSAP --code-only
|
||||
# ./scripts/sync_to_server.sh USER@HOST:/opt/HSAP --dms-only
|
||||
# ./scripts/sync_to_server.sh USER@HOST:/opt/HSAP --lane-only
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE="${1:?用法: $0 USER@HOST:/path/HSAP [--code-only|--dms-only|--lane-only]}"
|
||||
shift
|
||||
MODE="${1:-all}"
|
||||
|
||||
AS="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
DATA="$(cd "$AS/.." && pwd)"
|
||||
WS="$DATA/workspace"
|
||||
|
||||
RSYNC=(rsync -avh --info=progress2 --copy-links)
|
||||
|
||||
EXCLUDE=(
|
||||
--exclude 'vis_random_*'
|
||||
--exclude 'archive/'
|
||||
--exclude '__pycache__/'
|
||||
--exclude '.git/'
|
||||
--exclude 'runs/'
|
||||
--exclude 'log/'
|
||||
--exclude 'tmp/'
|
||||
--exclude 'node_modules/'
|
||||
--exclude 'platform/web/dist/'
|
||||
)
|
||||
|
||||
echo "HSAP: $AS"
|
||||
echo "实体 workspace: $WS"
|
||||
echo "远程: $REMOTE"
|
||||
|
||||
mkdir_remote() {
|
||||
ssh "${REMOTE%%:*}" "mkdir -p '${REMOTE#*:}'"
|
||||
}
|
||||
|
||||
sync_as_meta() {
|
||||
mkdir_remote
|
||||
"${RSYNC[@]}" \
|
||||
"$AS/as.py" \
|
||||
"$AS/workflow.registry.yaml" \
|
||||
"$AS/README.md" \
|
||||
"$AS/requirements.txt" \
|
||||
"${REMOTE}/"
|
||||
[[ -d "$AS/scripts" ]] && "${RSYNC[@]}" "$AS/scripts/" "${REMOTE}/scripts/"
|
||||
[[ -d "$AS/platform/as_platform" ]] && "${RSYNC[@]}" "$AS/platform/as_platform/" "${REMOTE}/platform/as_platform/"
|
||||
[[ -d "$AS/algorithms" ]] && "${RSYNC[@]}" --exclude 'code' "$AS/algorithms/" "${REMOTE}/algorithms/"
|
||||
[[ -f "$AS/algorithms/registry.yaml" ]] && "${RSYNC[@]}" "$AS/algorithms/registry.yaml" "${REMOTE}/algorithms/"
|
||||
}
|
||||
|
||||
sync_dms_code() {
|
||||
"${RSYNC[@]}" "${EXCLUDE[@]}" "$WS/DMS/Code/" "${REMOTE}/algorithms/dms_yolo/code/"
|
||||
}
|
||||
|
||||
sync_dms_data() {
|
||||
"${RSYNC[@]}" "${EXCLUDE[@]}" "$WS/DMS/DATASET/" "${REMOTE}/datasets/dms/"
|
||||
}
|
||||
|
||||
sync_lane_data() {
|
||||
"${RSYNC[@]}" "${EXCLUDE[@]}" "$WS/lane/" "${REMOTE}/datasets/lane/"
|
||||
}
|
||||
|
||||
sync_lane_code() {
|
||||
"${RSYNC[@]}" "${EXCLUDE[@]}" "$WS/LaneDection/Code/" "${REMOTE}/algorithms/lane_ufld/code/"
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
--code-only) sync_as_meta; sync_dms_code; sync_lane_code ;;
|
||||
--dms-only) sync_as_meta; sync_dms_code; sync_dms_data ;;
|
||||
--lane-only) sync_as_meta; sync_lane_data; sync_lane_code ;;
|
||||
*) sync_as_meta; sync_dms_code; sync_dms_data; sync_lane_data; sync_lane_code ;;
|
||||
esac
|
||||
|
||||
echo "完成: cd ${REMOTE#*:} && python as.py status"
|
||||
82
scripts/validate_dms_tasks.py
Normal file
82
scripts/validate_dms_tasks.py
Normal file
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""检查 DMS:yaml_active + active_packs 下数据。--task 仅查单任务。失败 exit 1。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
WORKSPACE = Path(__file__).resolve().parents[1]
|
||||
DATASET = (WORKSPACE / "datasets/dms").resolve()
|
||||
|
||||
|
||||
def load_context():
|
||||
reg = yaml.safe_load((DATASET / "datasets.registry.yaml").read_text())
|
||||
wf = yaml.safe_load((WORKSPACE / "workflow.registry.yaml").read_text())
|
||||
active = wf["projects"]["dms"]["active_packs"]
|
||||
sys.path.insert(0, str(DATASET / "scripts"))
|
||||
from pack_registry import resolve_pack_dir # noqa: E402
|
||||
return reg, active, resolve_pack_dir
|
||||
|
||||
|
||||
def check_yolo(data_dir: Path) -> list[str]:
|
||||
errs = []
|
||||
for sp in ("train", "val"):
|
||||
img_d = data_dir / "images" / sp
|
||||
lab_d = data_dir / "labels" / sp
|
||||
if not img_d.is_dir():
|
||||
errs.append(f"missing {img_d}")
|
||||
continue
|
||||
imgs = {p.stem for p in img_d.iterdir() if p.suffix.lower() in {".jpg", ".jpeg", ".png", ".bmp"}}
|
||||
labs = {p.stem for p in lab_d.glob("*.txt")} if lab_d.is_dir() else set()
|
||||
if imgs != labs:
|
||||
errs.append(f"{sp}: img/label mismatch ({len(imgs)} vs {len(labs)})")
|
||||
return errs
|
||||
|
||||
|
||||
def validate_task(task: str, tcfg: dict, active: list, resolve_pack_dir, errors: list) -> None:
|
||||
yml = DATASET / "manifests" / "yaml_active" / f"{task}.yaml"
|
||||
if not yml.is_file():
|
||||
errors.append(f"{task}: missing yaml_active {yml}")
|
||||
return
|
||||
for pack in active:
|
||||
base = resolve_pack_dir(DATASET, pack) / tcfg["task_dir"]
|
||||
if not base.is_dir():
|
||||
errors.append(f"{task}/{pack}: missing {base}")
|
||||
continue
|
||||
if tcfg["type"] in ("detect", "pose"):
|
||||
errors.extend(f"{task}/{pack}: {e}" for e in check_yolo(base))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--task")
|
||||
args = ap.parse_args()
|
||||
reg, active, resolve_pack_dir = load_context()
|
||||
print(f"active_packs: {active}")
|
||||
|
||||
tasks = reg["tasks"]
|
||||
if args.task:
|
||||
if args.task not in tasks:
|
||||
print(f"未知 task: {args.task}", file=sys.stderr)
|
||||
return 1
|
||||
tasks = {args.task: tasks[args.task]}
|
||||
|
||||
errors: list[str] = []
|
||||
for task, tcfg in tasks.items():
|
||||
print(f"\n{task} ({tcfg['type']})")
|
||||
validate_task(task, tcfg, active, resolve_pack_dir, errors)
|
||||
|
||||
if errors:
|
||||
print("\nFAILED:", file=sys.stderr)
|
||||
for e in errors:
|
||||
print(f" {e}", file=sys.stderr)
|
||||
return 1
|
||||
print("\nvalidate: OK")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
104
scripts/vendor_workspace.sh
Executable file
104
scripts/vendor_workspace.sh
Executable file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# 将 workspace 中的算法代码与数据集脚手架复制进 HSAP 仓库(供 Git 提交)
|
||||
#
|
||||
# 用法: bash scripts/vendor_workspace.sh
|
||||
# 大文件(图像、权重、训练 log)不会复制,见各 rsync --exclude
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
DATA="$(cd "$ROOT/.." && pwd)"
|
||||
WS="${AS_WORKSPACE_ROOT:-$DATA/workspace}"
|
||||
|
||||
if [[ ! -d "$WS" ]]; then
|
||||
echo "错误: workspace 不存在: $WS"
|
||||
echo "请设置 AS_WORKSPACE_ROOT 或保证 $DATA/workspace 存在"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "HSAP: $ROOT"
|
||||
echo "workspace: $WS"
|
||||
|
||||
RSYNC=(rsync -a --delete --no-group --no-owner --no-perms)
|
||||
|
||||
EXCLUDE_COMMON=(
|
||||
--exclude '__pycache__/'
|
||||
--exclude '*.pyc'
|
||||
--exclude '.git/'
|
||||
--exclude '.pytest_cache/'
|
||||
--exclude 'node_modules/'
|
||||
--exclude '*.egg-info/'
|
||||
)
|
||||
|
||||
# ── 移除旧软链,准备实体目录 ──
|
||||
for p in \
|
||||
"$ROOT/algorithms/dms_yolo/code" \
|
||||
"$ROOT/algorithms/lane_ufld/code" \
|
||||
"$ROOT/datasets/dms" \
|
||||
"$ROOT/datasets/lane"
|
||||
do
|
||||
if [[ -L "$p" ]]; then
|
||||
rm -f "$p"
|
||||
fi
|
||||
mkdir -p "$p"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo ">>> DMS YOLO 代码 → algorithms/dms_yolo/code/"
|
||||
"${RSYNC[@]}" "${EXCLUDE_COMMON[@]}" \
|
||||
--exclude 'runs/' \
|
||||
--exclude '*.pt' \
|
||||
--exclude '*.onnx' \
|
||||
--exclude '*.rknn' \
|
||||
--exclude '*.engine' \
|
||||
--exclude 'wandb/' \
|
||||
"$WS/DMS/Code/yolo26_rknn_ultralytics-main/" \
|
||||
"$ROOT/algorithms/dms_yolo/code/"
|
||||
|
||||
echo ">>> Lane UFLD 代码 → algorithms/lane_ufld/code/"
|
||||
"${RSYNC[@]}" "${EXCLUDE_COMMON[@]}" \
|
||||
--exclude 'log/' \
|
||||
--exclude 'tmp/' \
|
||||
--exclude 'runs/' \
|
||||
--exclude '*.pth' \
|
||||
--exclude '*.onnx' \
|
||||
--exclude 'wandb/' \
|
||||
"$WS/LaneDection/Code/" \
|
||||
"$ROOT/algorithms/lane_ufld/code/"
|
||||
|
||||
echo ">>> DMS 数据集脚手架 → datasets/dms/"
|
||||
"${RSYNC[@]}" "${EXCLUDE_COMMON[@]}" \
|
||||
--exclude 'packs/' \
|
||||
--exclude 'inbox/' \
|
||||
--exclude 'archive/' \
|
||||
--exclude 'sources/' \
|
||||
--exclude '*.jpg' --exclude '*.jpeg' --exclude '*.png' \
|
||||
--exclude '*.mp4' --exclude '*.avi' \
|
||||
"$WS/DMS/DATASET/" \
|
||||
"$ROOT/datasets/dms/"
|
||||
|
||||
echo ">>> Lane 数据集脚手架 → datasets/lane/"
|
||||
"${RSYNC[@]}" "${EXCLUDE_COMMON[@]}" \
|
||||
--exclude 'archive/' \
|
||||
--exclude 'DATASET/' \
|
||||
--exclude '*.jpg' --exclude '*.jpeg' --exclude '*.png' \
|
||||
--exclude '*.mp4' \
|
||||
"$WS/lane/" \
|
||||
"$ROOT/datasets/lane/"
|
||||
|
||||
# 标记:仓库内为实体文件,非软链
|
||||
cat > "$ROOT/manifests/repo_layout.json" <<EOF
|
||||
{
|
||||
"layout": "embedded",
|
||||
"vendored_at": "$(date -Iseconds)",
|
||||
"workspace_source": "$WS",
|
||||
"note": "算法代码与数据集脚手架已内嵌;大文件数据请挂载 workspace 或 rsync 到 datasets/"
|
||||
}
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "完成。请检查:"
|
||||
echo " ls algorithms/dms_yolo/code"
|
||||
echo " ls algorithms/lane_ufld/code/UFLD"
|
||||
echo " ls datasets/dms/scripts"
|
||||
echo ""
|
||||
echo "下一步: git add -A && git status"
|
||||
61
scripts/wait_for_db.py
Normal file
61
scripts/wait_for_db.py
Normal file
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
"""等待 PostgreSQL + Redis 就绪。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
from as_platform.config import IS_POSTGRES, JOB_EXECUTOR, REDIS_URL
|
||||
from as_platform.db.engine import check_connection
|
||||
from as_platform.redis.bus import ping_redis
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ok_db = True
|
||||
if IS_POSTGRES:
|
||||
ok_db = False
|
||||
for i in range(30):
|
||||
if check_connection():
|
||||
ok_db = True
|
||||
print("PostgreSQL 已就绪")
|
||||
break
|
||||
print(f"等待 PostgreSQL… ({i + 1}/30)")
|
||||
time.sleep(2)
|
||||
else:
|
||||
if check_connection():
|
||||
print("数据库已就绪")
|
||||
else:
|
||||
print("数据库连接失败", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
redis_required = JOB_EXECUTOR == "worker" or os.environ.get("AS_REQUIRE_REDIS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
if not REDIS_URL:
|
||||
print("Redis 未配置,跳过")
|
||||
if not ok_db:
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
for i in range(30):
|
||||
if ping_redis():
|
||||
print("Redis 已就绪")
|
||||
break
|
||||
print(f"等待 Redis… ({i + 1}/30)")
|
||||
time.sleep(2)
|
||||
else:
|
||||
if redis_required:
|
||||
print("Redis 连接超时", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print("警告: Redis 不可用,thread 模式仍可启动")
|
||||
|
||||
if not ok_db:
|
||||
print("PostgreSQL 连接超时", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
43
scripts/worker.py
Normal file
43
scripts/worker.py
Normal file
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Job Worker:从 Redis 队列取任务并执行(GPU/训练机可单独跑此脚本)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import signal
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "platform"))
|
||||
|
||||
from as_platform.config import JOB_EXECUTOR, JOB_QUEUE_KEY, REDIS_URL
|
||||
from as_platform.jobs.queue import _run_job
|
||||
from as_platform.redis.bus import ping_redis, pop_job
|
||||
|
||||
_running = True
|
||||
|
||||
|
||||
def _stop(*_):
|
||||
global _running
|
||||
_running = False
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if JOB_EXECUTOR != "worker":
|
||||
print(f"AS_JOB_EXECUTOR={JOB_EXECUTOR},worker 脚本建议设为 worker", file=sys.stderr)
|
||||
if not ping_redis():
|
||||
print(f"Redis 不可用: {REDIS_URL}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
signal.signal(signal.SIGINT, _stop)
|
||||
signal.signal(signal.SIGTERM, _stop)
|
||||
print(f"Worker 监听队列 {JOB_QUEUE_KEY} …")
|
||||
|
||||
while _running:
|
||||
job_id = pop_job(timeout=3)
|
||||
if job_id:
|
||||
print(f"执行 Job {job_id}")
|
||||
_run_job(job_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user