feat: initial HSAP platform
Huaxu Sentinel Active Safety Platform with embedded algorithm code, Docker Compose setup, and vendored dataset scaffolds for clone-and-run. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
0
platform/as_platform/jobs/__init__.py
Normal file
0
platform/as_platform/jobs/__init__.py
Normal file
145
platform/as_platform/jobs/queue.py
Normal file
145
platform/as_platform/jobs/queue.py
Normal file
@@ -0,0 +1,145 @@
|
||||
"""Job 队列（PostgreSQL + 可选 Redis Worker）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import JOB_EXECUTOR
|
||||
from as_platform.db.engine import session_scope
|
||||
from as_platform.db.models import Job
|
||||
|
||||
_executor_lock = threading.Lock()
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
return f"job-{datetime.now().strftime('%Y%m%d')}-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def enqueue_job(
|
||||
action: str,
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
approval_id: str | None = None,
|
||||
async_run: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
job_id = _new_id()
|
||||
with session_scope() as db:
|
||||
job = Job(
|
||||
id=job_id,
|
||||
status="queued",
|
||||
action=action,
|
||||
approval_id=approval_id,
|
||||
created_at=_now(),
|
||||
)
|
||||
job.set_params(params)
|
||||
db.add(job)
|
||||
|
||||
out = get_job(job_id) or {"id": job_id, "status": "queued", "action": action}
|
||||
|
||||
if not async_run:
|
||||
_run_job(job_id)
|
||||
return get_job(job_id) or out
|
||||
|
||||
if JOB_EXECUTOR == "worker":
|
||||
from as_platform.redis.bus import push_job
|
||||
|
||||
push_job(job_id)
|
||||
return out
|
||||
|
||||
threading.Thread(target=_run_job, args=(job_id,), daemon=True).start()
|
||||
return out
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Job, job_id)
|
||||
return rec.to_dict() if rec else None
|
||||
|
||||
|
||||
def list_jobs(status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with session_scope() as db:
|
||||
q = db.query(Job).order_by(Job.created_at.desc())
|
||||
if status:
|
||||
q = q.filter(Job.status == status)
|
||||
return [j.to_dict() for j in q.limit(limit).all()]
|
||||
|
||||
|
||||
def _patch(job_id: str, **fields: Any) -> dict[str, Any] | None:
|
||||
with session_scope() as db:
|
||||
rec = db.get(Job, job_id)
|
||||
if not rec:
|
||||
return None
|
||||
for k, v in fields.items():
|
||||
if k == "result" and isinstance(v, dict):
|
||||
rec.set_result(v)
|
||||
elif hasattr(rec, k):
|
||||
setattr(rec, k, v)
|
||||
db.flush()
|
||||
return rec.to_dict()
|
||||
|
||||
|
||||
def _compact_result(payload: Any) -> dict[str, Any]:
|
||||
if isinstance(payload, dict):
|
||||
out = dict(payload)
|
||||
else:
|
||||
out = {"value": payload}
|
||||
if "ok" not in out:
|
||||
out["ok"] = True
|
||||
for k in ("stdout", "stderr"):
|
||||
if isinstance(out.get(k), str):
|
||||
out[k] = out[k][-8000:]
|
||||
return out
|
||||
|
||||
|
||||
def _run_job(job_id: str) -> None:
|
||||
with _executor_lock:
|
||||
job = get_job(job_id)
|
||||
if not job or job.get("status") not in ("queued",):
|
||||
return
|
||||
_patch(job_id, status="running", started_at=_now())
|
||||
|
||||
from as_platform.agents.trace import trace_span
|
||||
from as_platform.jobs.runner import execute_action
|
||||
from as_platform.redis.bus import publish
|
||||
|
||||
publish("job.started", {"job_id": job_id, "action": job["action"]})
|
||||
|
||||
try:
|
||||
with trace_span("job_start", job_id=job_id, action=job["action"], approval_id=job.get("approval_id")):
|
||||
result = execute_action(job["action"], job.get("params") or {})
|
||||
persisted = _compact_result(result)
|
||||
_patch(
|
||||
job_id,
|
||||
status="succeeded",
|
||||
finished_at=_now(),
|
||||
result=persisted,
|
||||
)
|
||||
publish("job.succeeded", {"job_id": job_id})
|
||||
with trace_span("job_end", job_id=job_id, status="succeeded"):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "executed", persisted)
|
||||
except Exception as e:
|
||||
_patch(job_id, status="failed", finished_at=_now(), result={"ok": False, "error": str(e)})
|
||||
publish("job.failed", {"job_id": job_id, "error": str(e)})
|
||||
with trace_span("job_end", job_id=job_id, status="failed", error=str(e)):
|
||||
pass
|
||||
_sync_approval(job.get("approval_id"), "failed", {"error": str(e)})
|
||||
|
||||
|
||||
def _sync_approval(approval_id: str | None, status: str, result: dict) -> None:
|
||||
if not approval_id:
|
||||
return
|
||||
from as_platform.audit.queue import _update, _now as audit_now
|
||||
|
||||
_update(
|
||||
approval_id,
|
||||
status=status,
|
||||
executed_at=audit_now(),
|
||||
result=result if isinstance(result, dict) and "ok" in result else {"ok": status == "executed", **result},
|
||||
)
|
||||
159
platform/as_platform/jobs/runner.py
Normal file
159
platform/as_platform/jobs/runner.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""执行动作：优先引擎适配器，fallback as.py CLI。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
from as_platform.config import WORKSPACE, PLATFORM_DIR, LANE_DATA_VIZ_ENABLED
|
||||
|
||||
if str(WORKSPACE) not in sys.path:
|
||||
sys.path.insert(0, str(WORKSPACE))
|
||||
if str(PLATFORM_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(PLATFORM_DIR))
|
||||
|
||||
ML_PY = WORKSPACE / "as.py"
|
||||
AS_PY = ML_PY
|
||||
|
||||
LONG_ACTIONS = {"train_dms", "train_lane", "pipeline_dms", "eval_dms", "eval_lane", "visualize_dms", "visualize_lane"}
|
||||
|
||||
|
||||
def _run_ml(argv: list[str], timeout: int = 7200) -> dict[str, Any]:
|
||||
cmd = [sys.executable, str(ML_PY), *argv]
|
||||
proc = subprocess.run(cmd, cwd=str(WORKSPACE), capture_output=True, text=True, timeout=timeout)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"as.py 失败 (exit {proc.returncode}):\n{proc.stderr or proc.stdout}")
|
||||
return {"ok": True, "stdout": proc.stdout, "stderr": proc.stderr, "command": " ".join(cmd)}
|
||||
|
||||
|
||||
def execute_action(action: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
p = params or {}
|
||||
|
||||
if action == "train_dms":
|
||||
track = p.get("track", "platform")
|
||||
if track == "local":
|
||||
from algorithms.dms_yolo.adapter import train_local
|
||||
return train_local(p["task"], p.get("mode", "full"), p.get("config_overrides"))
|
||||
from algorithms.dms_yolo.adapter import train_platform
|
||||
return train_platform(p["task"], p.get("mode", "full"))
|
||||
|
||||
if action == "train_lane":
|
||||
track = p.get("track", "platform")
|
||||
if track == "local":
|
||||
from algorithms.lane_ufld.adapter import train_local
|
||||
return train_local(p.get("config_overrides"))
|
||||
from algorithms.lane_ufld.adapter import train_platform
|
||||
return train_platform()
|
||||
|
||||
if action == "train_dms_legacy":
|
||||
argv = ["train", "dms", p["task"]]
|
||||
if p.get("mode"):
|
||||
argv.extend(["--mode", str(p["mode"])])
|
||||
return _run_ml(argv, timeout=86400)
|
||||
|
||||
if action == "train_lane_legacy":
|
||||
return _run_ml(["train", "lane"], timeout=86400)
|
||||
|
||||
if action == "build_dms":
|
||||
argv = ["build", "dms", p["task"]]
|
||||
if p.get("pack"):
|
||||
argv.extend(["--pack", str(p["pack"])])
|
||||
if p.get("batch"):
|
||||
argv.extend(["--batch", str(p["batch"])])
|
||||
if p.get("all_sources"):
|
||||
argv.append("--all-sources")
|
||||
if p.get("dry_run"):
|
||||
argv.append("--dry-run")
|
||||
if p.get("skip_validate"):
|
||||
argv.append("--skip-validate")
|
||||
if p.get("no_refresh"):
|
||||
argv.append("--no-refresh")
|
||||
return _run_ml(argv)
|
||||
|
||||
if action == "build_lane":
|
||||
return _run_ml(["build", "lane"])
|
||||
|
||||
if action == "enable_pack":
|
||||
return _run_ml(["enable", p["project"], p["pack"]])
|
||||
|
||||
if action == "disable_pack":
|
||||
return _run_ml(["disable", p["project"], p["pack"]])
|
||||
|
||||
if action == "eval_dms":
|
||||
argv = ["eval", "dms", p["task"]]
|
||||
if p.get("save_candidate"):
|
||||
argv.append("--save-candidate")
|
||||
if p.get("weights"):
|
||||
argv.extend(["--weights", str(p["weights"])])
|
||||
return _run_ml(argv, timeout=3600)
|
||||
|
||||
if action == "eval_lane":
|
||||
from algorithms.lane_ufld.adapter import eval_task
|
||||
|
||||
return eval_task(
|
||||
model_path=p.get("model_path"),
|
||||
data_root=p.get("data_root"),
|
||||
test_list=p.get("test_list", "list/test_gt.txt"),
|
||||
)
|
||||
|
||||
if action == "visualize_dms":
|
||||
from algorithms.dms_yolo.adapter import visualize_task
|
||||
|
||||
return visualize_task(
|
||||
p["task"],
|
||||
weights=p.get("weights"),
|
||||
)
|
||||
|
||||
if action == "visualize_lane":
|
||||
if not LANE_DATA_VIZ_ENABLED:
|
||||
raise RuntimeError("车道线数据可视化暂未开放")
|
||||
from algorithms.lane_ufld.adapter import visualize_task
|
||||
|
||||
return visualize_task(
|
||||
model_path=p.get("model_path"),
|
||||
data_root=p.get("data_root"),
|
||||
test_list=p.get("test_list", "list/test_gt.txt"),
|
||||
)
|
||||
|
||||
if action == "promote_dms":
|
||||
argv = ["promote", "dms", p["task"]]
|
||||
if p.get("force"):
|
||||
argv.append("--force")
|
||||
return _run_ml(argv)
|
||||
|
||||
if action == "pipeline_dms":
|
||||
argv = ["pipeline", "dms", p["task"], "--pack", str(p.get("pack", "dms_v2"))]
|
||||
if p.get("batch"):
|
||||
argv.extend(["--batch", str(p["batch"])])
|
||||
if p.get("all_sources"):
|
||||
argv.append("--all-sources")
|
||||
if p.get("train"):
|
||||
argv.append("--train")
|
||||
if p.get("dry_run"):
|
||||
argv.append("--dry-run")
|
||||
return _run_ml(argv, timeout=86400)
|
||||
|
||||
if action == "register_batch":
|
||||
from as_platform.data.core import register_batch
|
||||
|
||||
register_batch(
|
||||
None, p["project"], p.get("task"), p["batch"],
|
||||
pack=p.get("pack"), stage=p.get("stage", "returned"),
|
||||
engineer=p.get("engineer"), location=p.get("location", "inbox"),
|
||||
)
|
||||
return {"ok": True, "stdout": "register_batch ok", "stderr": ""}
|
||||
|
||||
if action == "analyze_uploaded_dataset":
|
||||
from as_platform.data.lake import analyze_uploaded_candidate
|
||||
|
||||
candidate_id = p["candidate_id"]
|
||||
result = analyze_uploaded_candidate(candidate_id)
|
||||
return {
|
||||
"ok": True,
|
||||
"stdout": json.dumps(result, ensure_ascii=False),
|
||||
"stderr": "",
|
||||
"result": result,
|
||||
}
|
||||
|
||||
raise ValueError(f"未实现执行: {action}")
|
||||
Reference in New Issue
Block a user