单目3D初始代码
This commit is contained in:
zhao.zhu
721
tools/feishu_project/run_issue_data_inference.py
Executable file
721
tools/feishu_project/run_issue_data_inference.py
Executable file
@@ -0,0 +1,721 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Batch inference runner for downloaded Feishu issue data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_DOWNLOAD_ROOT = Path("/data1/dongying/Mono3d/G1Q3/feishu_project/downloaded_issue_data")
|
||||
DEFAULT_OUTPUT_ROOT = Path("/data1/dongying/Mono3d/G1Q3/feishu_project/inference_issue_data")
|
||||
DEFAULT_INFERENCE_SCRIPT = ROOT / "tools" / "model_inference" / "core" / "run_two_roi_exported_onnx_infer.py"
|
||||
DEFAULT_PYTHON_BIN = Path("/deeplearning_team/ydong/dongying/miniconda/envs/dev/bin/python")
|
||||
ISSUE_DIR_RE = re.compile(r"^issue_(\d+)$")
|
||||
SIGNED_INTEGER_RE = re.compile(r"^[+-]?\d+$")
|
||||
FRAME_ID_CAMERA4_RE = re.compile(r"camera4\s*:\s*([+-]?\d+)", re.IGNORECASE)
|
||||
FRAME_ID_ANY_CAMERA_RE = re.compile(r"(camera\d+)\s*:\s*([+-]?\d+)", re.IGNORECASE)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetFrame:
|
||||
camera: str
|
||||
frame_id: int
|
||||
raw_text: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InferenceCase:
|
||||
issue_id: int
|
||||
issue_dir: Path
|
||||
case_dir: Path
|
||||
camera4_bin: Path
|
||||
relative_case_dir: Path
|
||||
output_dir: Path
|
||||
frame_window_source: str
|
||||
target_frame_text: str | None = None
|
||||
target_frame_id: int | None = None
|
||||
requested_frame_index_start: int | None = None
|
||||
requested_frame_index_end: int | None = None
|
||||
requested_frame_id_start: int | None = None
|
||||
requested_frame_id_end: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaseResult:
|
||||
issue_id: int
|
||||
issue_dir: str
|
||||
case_dir: str
|
||||
camera4_bin: str
|
||||
relative_case_dir: str
|
||||
output_dir: str
|
||||
status: str
|
||||
detail: str
|
||||
command: list[str]
|
||||
log_path: str | None = None
|
||||
frame_window_source: str | None = None
|
||||
target_frame_text: str | None = None
|
||||
target_frame_id: int | None = None
|
||||
requested_frame_index_start: int | None = None
|
||||
requested_frame_index_end: int | None = None
|
||||
requested_frame_id_start: int | None = None
|
||||
requested_frame_id_end: int | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"issue_id": self.issue_id,
|
||||
"issue_dir": self.issue_dir,
|
||||
"case_dir": self.case_dir,
|
||||
"camera4_bin": self.camera4_bin,
|
||||
"relative_case_dir": self.relative_case_dir,
|
||||
"output_dir": self.output_dir,
|
||||
"status": self.status,
|
||||
"detail": self.detail,
|
||||
"command": self.command,
|
||||
"log_path": self.log_path,
|
||||
"frame_window_source": self.frame_window_source,
|
||||
"target_frame_text": self.target_frame_text,
|
||||
"target_frame_id": self.target_frame_id,
|
||||
"requested_frame_index_start": self.requested_frame_index_start,
|
||||
"requested_frame_index_end": self.requested_frame_index_end,
|
||||
"requested_frame_id_start": self.requested_frame_id_start,
|
||||
"requested_frame_id_end": self.requested_frame_id_end,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run exported-model inference on all downloaded issue-data camera4.bin cases."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--download-root",
|
||||
default=str(DEFAULT_DOWNLOAD_ROOT),
|
||||
help="Root directory produced by download_issue_data.py.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
default=str(DEFAULT_OUTPUT_ROOT),
|
||||
help="Root directory where inference outputs will be written.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--manifest-path",
|
||||
default="",
|
||||
help="Optional explicit manifest JSON path. Defaults to <output-root>/inference_manifest.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--python-bin",
|
||||
default=str(DEFAULT_PYTHON_BIN),
|
||||
help="Python interpreter used to launch run_two_roi_exported_onnx_infer.py.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--inference-script",
|
||||
default=str(DEFAULT_INFERENCE_SCRIPT),
|
||||
help="Path to run_two_roi_exported_onnx_infer.py.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--issue-json",
|
||||
default="",
|
||||
help="Optional Feishu issue export JSON used to resolve 问题发生frameid windows.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--issue-id",
|
||||
action="append",
|
||||
dest="issue_ids",
|
||||
type=int,
|
||||
help="Optional issue id filter. Can be repeated.",
|
||||
)
|
||||
parser.add_argument("--video-stride", type=int, default=1)
|
||||
parser.add_argument("--max-images", type=int, default=0)
|
||||
parser.add_argument("--frame-index-start", type=int, default=None)
|
||||
parser.add_argument("--frame-index-end", type=int, default=None)
|
||||
parser.add_argument("--frame-id-start", type=int, default=None)
|
||||
parser.add_argument("--frame-id-end", type=int, default=None)
|
||||
parser.add_argument("--target-frame-id", type=int, default=None)
|
||||
parser.add_argument("--frame-before", type=int, default=100)
|
||||
parser.add_argument("--frame-after", type=int, default=100)
|
||||
parser.add_argument("--use-issue-frame-window", action="store_true")
|
||||
parser.add_argument(
|
||||
"--missing-issue-frame-policy",
|
||||
choices=("full", "skip"),
|
||||
default="full",
|
||||
help="How to handle cases without a usable 问题发生frameid when --use-issue-frame-window is enabled.",
|
||||
)
|
||||
parser.add_argument("--exported-model", type=str, default="")
|
||||
parser.add_argument("--device", type=str, default="")
|
||||
parser.add_argument("--providers", nargs="*", default=None)
|
||||
parser.add_argument("--enable-attr", action="store_true")
|
||||
parser.add_argument("--enable-cross-class-merge-prior", action="store_true")
|
||||
parser.add_argument("--save-aggregate-predictions", action="store_true")
|
||||
parser.add_argument(
|
||||
"--inference-arg",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Extra argument forwarded to run_two_roi_exported_onnx_infer.py. Can be repeated.",
|
||||
)
|
||||
parser.add_argument("--skip-existing", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.frame_before < 0:
|
||||
parser.error("--frame-before must be greater than or equal to 0")
|
||||
if args.frame_after < 0:
|
||||
parser.error("--frame-after must be greater than or equal to 0")
|
||||
if (
|
||||
args.frame_index_start is not None
|
||||
and args.frame_index_end is not None
|
||||
and args.frame_index_start > args.frame_index_end
|
||||
):
|
||||
parser.error("--frame-index-start must be less than or equal to --frame-index-end")
|
||||
if (
|
||||
args.frame_id_start is not None
|
||||
and args.frame_id_end is not None
|
||||
and args.frame_id_start > args.frame_id_end
|
||||
):
|
||||
parser.error("--frame-id-start must be less than or equal to --frame-id-end")
|
||||
if args.use_issue_frame_window and not args.issue_json:
|
||||
parser.error("--issue-json is required when --use-issue-frame-window is enabled")
|
||||
return args
|
||||
|
||||
|
||||
def ensure_dir(path: Path, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
return
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def log_progress(message: str) -> None:
|
||||
timestamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S")
|
||||
print(f"[run_issue_data_inference {timestamp}] {message}", flush=True)
|
||||
|
||||
|
||||
def compact_text(value: object, max_len: int = 96) -> str:
|
||||
text = "" if value is None else str(value).strip()
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
return f"{text[: max_len - 3]}..."
|
||||
|
||||
|
||||
def parse_issue_id_from_path(path: Path) -> int | None:
|
||||
for part in path.parts:
|
||||
match = ISSUE_DIR_RE.match(part)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def load_issue_items(path: Path) -> list[dict]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
return payload["items"]
|
||||
|
||||
|
||||
def parse_target_frame(frame_text: object) -> tuple[TargetFrame | None, str]:
|
||||
text = "" if frame_text is None else str(frame_text).strip()
|
||||
if not text:
|
||||
return None, "missing 问题发生frameid"
|
||||
camera4_match = FRAME_ID_CAMERA4_RE.search(text)
|
||||
if camera4_match:
|
||||
frame_id = int(camera4_match.group(1))
|
||||
if frame_id <= 0:
|
||||
return None, f"non-positive 问题发生frameid: {text!r}"
|
||||
return TargetFrame(camera="camera4", frame_id=frame_id, raw_text=text), ""
|
||||
if SIGNED_INTEGER_RE.fullmatch(text):
|
||||
frame_id = int(text)
|
||||
if frame_id <= 0:
|
||||
return None, f"non-positive 问题发生frameid: {text!r}"
|
||||
return TargetFrame(camera="any", frame_id=frame_id, raw_text=text), ""
|
||||
any_camera_match = FRAME_ID_ANY_CAMERA_RE.search(text)
|
||||
if any_camera_match:
|
||||
frame_id = int(any_camera_match.group(2))
|
||||
if frame_id <= 0:
|
||||
return None, f"non-positive 问题发生frameid: {text!r}"
|
||||
return TargetFrame(camera=any_camera_match.group(1).lower(), frame_id=frame_id, raw_text=text), ""
|
||||
return None, f"unparseable 问题发生frameid: {text!r}"
|
||||
|
||||
|
||||
def build_issue_item_lookup(issue_json: Path) -> dict[int, dict]:
|
||||
return {int(item["id"]): item for item in load_issue_items(issue_json)}
|
||||
|
||||
|
||||
def has_manual_frame_window(args: argparse.Namespace) -> bool:
|
||||
return any(
|
||||
value is not None
|
||||
for value in (
|
||||
args.frame_index_start,
|
||||
args.frame_index_end,
|
||||
args.frame_id_start,
|
||||
args.frame_id_end,
|
||||
args.target_frame_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def build_case_result(
|
||||
case: InferenceCase,
|
||||
*,
|
||||
status: str,
|
||||
detail: str,
|
||||
command: list[str] | None = None,
|
||||
log_path: str | None = None,
|
||||
) -> CaseResult:
|
||||
return CaseResult(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=str(case.issue_dir),
|
||||
case_dir=str(case.case_dir),
|
||||
camera4_bin=str(case.camera4_bin),
|
||||
relative_case_dir=str(case.relative_case_dir),
|
||||
output_dir=str(case.output_dir),
|
||||
status=status,
|
||||
detail=detail,
|
||||
command=command or [],
|
||||
log_path=log_path,
|
||||
frame_window_source=case.frame_window_source,
|
||||
target_frame_text=case.target_frame_text,
|
||||
target_frame_id=case.target_frame_id,
|
||||
requested_frame_index_start=case.requested_frame_index_start,
|
||||
requested_frame_index_end=case.requested_frame_index_end,
|
||||
requested_frame_id_start=case.requested_frame_id_start,
|
||||
requested_frame_id_end=case.requested_frame_id_end,
|
||||
)
|
||||
|
||||
|
||||
def discover_cases(download_root: Path, output_root: Path, issue_filter: set[int] | None) -> list[InferenceCase]:
|
||||
cases: list[InferenceCase] = []
|
||||
for camera4_bin in sorted(download_root.rglob("camera4.bin")):
|
||||
if camera4_bin.parent.name != "sigmastar.1":
|
||||
continue
|
||||
try:
|
||||
relative_camera = camera4_bin.relative_to(download_root)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
issue_id = parse_issue_id_from_path(relative_camera)
|
||||
if issue_id is None:
|
||||
continue
|
||||
if issue_filter and issue_id not in issue_filter:
|
||||
continue
|
||||
|
||||
case_dir = camera4_bin.parent.parent
|
||||
relative_case_dir = case_dir.relative_to(download_root)
|
||||
issue_dir = download_root / f"issue_{issue_id}"
|
||||
output_dir = output_root / relative_case_dir
|
||||
cases.append(
|
||||
InferenceCase(
|
||||
issue_id=issue_id,
|
||||
issue_dir=issue_dir,
|
||||
case_dir=case_dir,
|
||||
camera4_bin=camera4_bin,
|
||||
relative_case_dir=relative_case_dir,
|
||||
output_dir=output_dir,
|
||||
frame_window_source="full",
|
||||
)
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
def apply_frame_windows(
|
||||
cases: list[InferenceCase],
|
||||
args: argparse.Namespace,
|
||||
issue_item_lookup: dict[int, dict],
|
||||
) -> tuple[list[InferenceCase], list[CaseResult]]:
|
||||
prepared_cases: list[InferenceCase] = []
|
||||
skipped_results: list[CaseResult] = []
|
||||
|
||||
for case in cases:
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source="full",
|
||||
)
|
||||
|
||||
if has_manual_frame_window(args):
|
||||
requested_frame_id_start = args.frame_id_start
|
||||
requested_frame_id_end = args.frame_id_end
|
||||
if args.target_frame_id is not None:
|
||||
if requested_frame_id_start is None:
|
||||
requested_frame_id_start = max(0, args.target_frame_id - args.frame_before)
|
||||
if requested_frame_id_end is None:
|
||||
requested_frame_id_end = args.target_frame_id + args.frame_after
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source="manual",
|
||||
target_frame_id=args.target_frame_id,
|
||||
requested_frame_index_start=args.frame_index_start,
|
||||
requested_frame_index_end=args.frame_index_end,
|
||||
requested_frame_id_start=requested_frame_id_start,
|
||||
requested_frame_id_end=requested_frame_id_end,
|
||||
)
|
||||
prepared_cases.append(prepared_case)
|
||||
continue
|
||||
|
||||
if not args.use_issue_frame_window:
|
||||
prepared_cases.append(prepared_case)
|
||||
continue
|
||||
|
||||
item = issue_item_lookup.get(case.issue_id)
|
||||
if item is None:
|
||||
if args.missing_issue_frame_policy == "skip":
|
||||
skipped_results.append(
|
||||
build_case_result(
|
||||
prepared_case,
|
||||
status="skipped_missing_issue_record",
|
||||
detail="issue id not found in issue_json",
|
||||
)
|
||||
)
|
||||
continue
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source="full_missing_issue_record",
|
||||
)
|
||||
prepared_cases.append(prepared_case)
|
||||
continue
|
||||
|
||||
target_frame, target_frame_error = parse_target_frame(item.get("问题发生frameid"))
|
||||
if target_frame is None:
|
||||
if args.missing_issue_frame_policy == "skip":
|
||||
skipped_results.append(
|
||||
build_case_result(
|
||||
prepared_case,
|
||||
status="skipped_missing_issue_frame",
|
||||
detail=target_frame_error,
|
||||
)
|
||||
)
|
||||
continue
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source=(
|
||||
"full_missing_issue_frame"
|
||||
if target_frame_error.startswith("missing ")
|
||||
else "full_invalid_issue_frame"
|
||||
),
|
||||
)
|
||||
prepared_cases.append(prepared_case)
|
||||
continue
|
||||
|
||||
if target_frame.camera not in {"camera4", "any"}:
|
||||
if args.missing_issue_frame_policy == "skip":
|
||||
skipped_results.append(
|
||||
build_case_result(
|
||||
prepared_case,
|
||||
status="skipped_unsupported_issue_camera",
|
||||
detail=f"unsupported target camera for window inference: {target_frame.camera}",
|
||||
)
|
||||
)
|
||||
continue
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source="full_unsupported_issue_camera",
|
||||
target_frame_text=target_frame.raw_text,
|
||||
)
|
||||
prepared_cases.append(prepared_case)
|
||||
continue
|
||||
|
||||
prepared_case = InferenceCase(
|
||||
issue_id=case.issue_id,
|
||||
issue_dir=case.issue_dir,
|
||||
case_dir=case.case_dir,
|
||||
camera4_bin=case.camera4_bin,
|
||||
relative_case_dir=case.relative_case_dir,
|
||||
output_dir=case.output_dir,
|
||||
frame_window_source="issue_frame",
|
||||
target_frame_text=target_frame.raw_text,
|
||||
target_frame_id=target_frame.frame_id,
|
||||
requested_frame_id_start=max(0, target_frame.frame_id - args.frame_before),
|
||||
requested_frame_id_end=target_frame.frame_id + args.frame_after,
|
||||
)
|
||||
prepared_cases.append(prepared_case)
|
||||
|
||||
return prepared_cases, skipped_results
|
||||
|
||||
|
||||
def has_existing_outputs(output_dir: Path) -> bool:
|
||||
if (output_dir / "predictions_merged.json").is_file():
|
||||
return True
|
||||
merge_json_dir = output_dir / "predictions" / "merge"
|
||||
if merge_json_dir.is_dir():
|
||||
for child in merge_json_dir.iterdir():
|
||||
if child.is_file():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_command(case: InferenceCase, args: argparse.Namespace) -> list[str]:
|
||||
command = [
|
||||
str(Path(args.python_bin).resolve()),
|
||||
str(Path(args.inference_script).resolve()),
|
||||
"--video-case-dir",
|
||||
str(case.camera4_bin),
|
||||
"--output-dir",
|
||||
str(case.output_dir),
|
||||
"--video-stride",
|
||||
str(args.video_stride),
|
||||
]
|
||||
if args.max_images > 0:
|
||||
command.extend(["--max-images", str(args.max_images)])
|
||||
if case.target_frame_id is not None:
|
||||
command.extend(["--target-frame-id", str(case.target_frame_id)])
|
||||
if case.requested_frame_index_start is not None:
|
||||
command.extend(["--frame-index-start", str(case.requested_frame_index_start)])
|
||||
if case.requested_frame_index_end is not None:
|
||||
command.extend(["--frame-index-end", str(case.requested_frame_index_end)])
|
||||
if case.requested_frame_id_start is not None:
|
||||
command.extend(["--frame-id-start", str(case.requested_frame_id_start)])
|
||||
if case.requested_frame_id_end is not None:
|
||||
command.extend(["--frame-id-end", str(case.requested_frame_id_end)])
|
||||
if args.exported_model:
|
||||
command.extend(["--exported-model", args.exported_model])
|
||||
if args.device:
|
||||
command.extend(["--device", args.device])
|
||||
if args.providers:
|
||||
command.extend(["--providers", *args.providers])
|
||||
if args.enable_attr:
|
||||
command.append("--enable-attr")
|
||||
if args.enable_cross_class_merge_prior:
|
||||
command.append("--enable-cross-class-merge-prior")
|
||||
if args.save_aggregate_predictions:
|
||||
command.append("--save-aggregate-predictions")
|
||||
for extra_arg in args.inference_arg:
|
||||
command.append(extra_arg)
|
||||
return command
|
||||
|
||||
|
||||
def write_case_status_files(case: InferenceCase, command: list[str], log_text: str, dry_run: bool) -> str:
|
||||
status_dir = case.output_dir / "_status"
|
||||
if dry_run:
|
||||
return str(status_dir / "inference.log")
|
||||
ensure_dir(status_dir, dry_run=False)
|
||||
(status_dir / "command.txt").write_text(" ".join(command) + "\n", encoding="utf-8")
|
||||
log_path = status_dir / "inference.log"
|
||||
log_path.write_text(log_text, encoding="utf-8")
|
||||
return str(log_path)
|
||||
|
||||
|
||||
def summarize_process_output(completed: subprocess.CompletedProcess[str]) -> str:
|
||||
stdout = completed.stdout.strip()
|
||||
stderr = completed.stderr.strip()
|
||||
if completed.returncode == 0:
|
||||
if stdout:
|
||||
return stdout.splitlines()[-1]
|
||||
return "inference completed"
|
||||
if stderr:
|
||||
return stderr.splitlines()[-1]
|
||||
if stdout:
|
||||
return stdout.splitlines()[-1]
|
||||
return f"inference failed with return code {completed.returncode}"
|
||||
|
||||
|
||||
def run_case(case: InferenceCase, args: argparse.Namespace) -> CaseResult:
|
||||
command = build_command(case, args)
|
||||
|
||||
if args.skip_existing and has_existing_outputs(case.output_dir):
|
||||
return build_case_result(
|
||||
case,
|
||||
status="skipped_existing",
|
||||
detail="existing inference outputs found",
|
||||
command=command,
|
||||
log_path=None,
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
return build_case_result(
|
||||
case,
|
||||
status="planned",
|
||||
detail="would run inference",
|
||||
command=command,
|
||||
log_path=str(case.output_dir / "_status" / "inference.log"),
|
||||
)
|
||||
|
||||
ensure_dir(case.output_dir, dry_run=False)
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=str(ROOT),
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
combined_log = "\n".join(
|
||||
part for part in (completed.stdout.strip(), completed.stderr.strip()) if part
|
||||
)
|
||||
log_path = write_case_status_files(case, command, combined_log + ("\n" if combined_log else ""), dry_run=False)
|
||||
status = "success" if completed.returncode == 0 else "failed"
|
||||
detail = summarize_process_output(completed)
|
||||
return build_case_result(
|
||||
case,
|
||||
status=status,
|
||||
detail=detail,
|
||||
command=command,
|
||||
log_path=log_path,
|
||||
)
|
||||
|
||||
|
||||
def build_manifest(
|
||||
args: argparse.Namespace,
|
||||
download_root: Path,
|
||||
output_root: Path,
|
||||
discovered_cases: list[InferenceCase],
|
||||
results: list[CaseResult],
|
||||
) -> dict:
|
||||
summary: dict[str, int] = {}
|
||||
for result in results:
|
||||
summary[result.status] = summary.get(result.status, 0) + 1
|
||||
return {
|
||||
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"download_root": str(download_root),
|
||||
"output_root": str(output_root),
|
||||
"python_bin": str(Path(args.python_bin).resolve()),
|
||||
"inference_script": str(Path(args.inference_script).resolve()),
|
||||
"dry_run": args.dry_run,
|
||||
"skip_existing": args.skip_existing,
|
||||
"issue_filter": args.issue_ids or [],
|
||||
"issue_json": args.issue_json,
|
||||
"use_issue_frame_window": args.use_issue_frame_window,
|
||||
"missing_issue_frame_policy": args.missing_issue_frame_policy,
|
||||
"video_stride": args.video_stride,
|
||||
"max_images": args.max_images,
|
||||
"frame_index_start": args.frame_index_start,
|
||||
"frame_index_end": args.frame_index_end,
|
||||
"frame_id_start": args.frame_id_start,
|
||||
"frame_id_end": args.frame_id_end,
|
||||
"target_frame_id": args.target_frame_id,
|
||||
"frame_before": args.frame_before,
|
||||
"frame_after": args.frame_after,
|
||||
"exported_model": args.exported_model,
|
||||
"device": args.device,
|
||||
"providers": args.providers or [],
|
||||
"enable_attr": args.enable_attr,
|
||||
"enable_cross_class_merge_prior": args.enable_cross_class_merge_prior,
|
||||
"save_aggregate_predictions": args.save_aggregate_predictions,
|
||||
"inference_args": args.inference_arg,
|
||||
"total_cases": len(discovered_cases),
|
||||
"summary": summary,
|
||||
"cases": [result.to_dict() for result in results],
|
||||
}
|
||||
|
||||
|
||||
def print_summary(manifest: dict) -> None:
|
||||
print(f"download_root: {manifest['download_root']}")
|
||||
print(f"output_root: {manifest['output_root']}")
|
||||
print(f"dry_run: {manifest['dry_run']}")
|
||||
if manifest["use_issue_frame_window"]:
|
||||
print(
|
||||
"issue_frame_window: "
|
||||
f"enabled issue_json={manifest['issue_json']} "
|
||||
f"before={manifest['frame_before']} after={manifest['frame_after']} "
|
||||
f"missing_policy={manifest['missing_issue_frame_policy']}"
|
||||
)
|
||||
elif (
|
||||
manifest["target_frame_id"] is not None
|
||||
or manifest["frame_id_start"] is not None
|
||||
or manifest["frame_id_end"] is not None
|
||||
or manifest["frame_index_start"] is not None
|
||||
or manifest["frame_index_end"] is not None
|
||||
):
|
||||
print(
|
||||
"manual_frame_window: "
|
||||
f"target_frame_id={manifest['target_frame_id']} "
|
||||
f"frame_id=[{manifest['frame_id_start'] if manifest['frame_id_start'] is not None else '-inf'}, "
|
||||
f"{manifest['frame_id_end'] if manifest['frame_id_end'] is not None else '+inf'}] "
|
||||
f"frame_index=[{manifest['frame_index_start'] if manifest['frame_index_start'] is not None else '-inf'}, "
|
||||
f"{manifest['frame_index_end'] if manifest['frame_index_end'] is not None else '+inf'}]"
|
||||
)
|
||||
print(f"total_cases: {manifest['total_cases']}")
|
||||
for status, count in sorted(manifest["summary"].items()):
|
||||
print(f"{status}: {count}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
download_root = Path(args.download_root).resolve()
|
||||
output_root = Path(args.output_root).resolve()
|
||||
manifest_path = (
|
||||
Path(args.manifest_path).resolve()
|
||||
if args.manifest_path
|
||||
else output_root / "inference_manifest.json"
|
||||
)
|
||||
|
||||
issue_filter = set(args.issue_ids) if args.issue_ids else None
|
||||
cases = discover_cases(download_root, output_root, issue_filter)
|
||||
if not cases:
|
||||
raise FileNotFoundError(f"No */sigmastar.1/camera4.bin cases found under {download_root}")
|
||||
|
||||
issue_item_lookup: dict[int, dict] = {}
|
||||
if args.use_issue_frame_window:
|
||||
issue_item_lookup = build_issue_item_lookup(Path(args.issue_json).resolve())
|
||||
|
||||
prepared_cases, skipped_results = apply_frame_windows(cases, args, issue_item_lookup)
|
||||
log_progress(
|
||||
"cases discovered: "
|
||||
f"total={len(cases)} runnable={len(prepared_cases)} skipped_precheck={len(skipped_results)}"
|
||||
)
|
||||
for skipped_result in skipped_results:
|
||||
log_progress(
|
||||
f"skip issue_{skipped_result.issue_id} {compact_text(skipped_result.relative_case_dir)}: "
|
||||
f"{skipped_result.status} ({compact_text(skipped_result.detail, max_len=72)})"
|
||||
)
|
||||
|
||||
ensure_dir(output_root, dry_run=args.dry_run)
|
||||
results = list(skipped_results)
|
||||
total_prepared_cases = len(prepared_cases)
|
||||
for index, case in enumerate(prepared_cases, start=1):
|
||||
log_progress(
|
||||
f"[{index}/{total_prepared_cases}] issue_{case.issue_id} "
|
||||
f"{compact_text(case.relative_case_dir)}: start (window={case.frame_window_source})"
|
||||
)
|
||||
case_result = run_case(case, args)
|
||||
results.append(case_result)
|
||||
log_progress(
|
||||
f"[{index}/{total_prepared_cases}] issue_{case.issue_id} "
|
||||
f"{compact_text(case.relative_case_dir)}: "
|
||||
f"{case_result.status} ({compact_text(case_result.detail, max_len=72)})"
|
||||
)
|
||||
manifest = build_manifest(args, download_root, output_root, cases, results)
|
||||
|
||||
if not args.dry_run:
|
||||
ensure_dir(manifest_path.parent, dry_run=False)
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print_summary(manifest)
|
||||
if args.dry_run:
|
||||
print(f"manifest (not written in dry-run): {manifest_path}")
|
||||
else:
|
||||
print(f"manifest: {manifest_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user