8010 lines
362 KiB
Python
Executable File
8010 lines
362 KiB
Python
Executable File
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import html as html_lib
|
||
import json
|
||
import math
|
||
import os
|
||
import random
|
||
import sys
|
||
import time
|
||
from collections import Counter, defaultdict
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
from typing import Any, Iterable, Optional
|
||
|
||
import cv2
|
||
import numpy as np
|
||
import torch
|
||
import yaml
|
||
|
||
FILE = Path(__file__).resolve()
|
||
ROOT = FILE.parents[2]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.append(str(ROOT))
|
||
|
||
from ultralytics.data.ground3d_augment import normalize_roi_depth, parse_ground_3d_label_file, read_calib_from_path, remap_labels_to_roi
|
||
from ultralytics.engine.validator import BaseValidator
|
||
from ultralytics.utils.plotting_3d import (
|
||
EDGE_YAW_VALID_VISIBILITY_SCORE_THRESH,
|
||
FACE_COLORS,
|
||
classify_edge_yaw_prediction_bucket,
|
||
collect_face_bottom_edges,
|
||
create_bev_image,
|
||
decode_3d_target,
|
||
decode_multi_visible_face_yaw_from_gt,
|
||
draw_3d_box,
|
||
extract_3d_attrs_from_gt,
|
||
extract_3d_attrs_from_prediction,
|
||
face_center_from_corners,
|
||
get_gt_cut_state,
|
||
get_pred_cut_state,
|
||
is_gt_cut_object,
|
||
project_3d_to_2d_with_distortion,
|
||
rebuild_box_corners_for_visualization,
|
||
select_gt_visible_faces,
|
||
)
|
||
from ultralytics.utils.metrics import ConfusionMatrix, ap_per_class, smooth
|
||
from tools.pdcl_inference.two_roi_inference import (
|
||
filter_prediction_outputs,
|
||
iter_batches,
|
||
prepare_roi_image,
|
||
decode_prepared_roi_predictions,
|
||
run_model_for_prepared_roi_batch,
|
||
)
|
||
from tools.pdcl_inference.run_batch_two_roi_infer import (
|
||
add_two_roi_inference_args,
|
||
build_two_roi_inference_context_from_args,
|
||
populate_two_roi_inference_args,
|
||
)
|
||
|
||
|
||
DEFAULT_OUTPUT_ROOT = FILE.parent / "validation_analysis" / "report_{}".format(datetime.now().strftime("%Y%m%d_%H%M%S"))
|
||
FACE_NAMES = {0: "front", 1: "rear", 2: "left", 3: "right"}
|
||
FACE_SELECTION_LABEL_ORDER = ("cut_in", "cut_out", "front", "rear", "left", "right")
|
||
FAKE_CLASS_LABEL_ORDER = ("non_fake", "car_fake", "bicyclist_fake", "pedestrian_fake", "car", "bicycle", "pedestrian")
|
||
OCCLUSION_BINARY_LABEL_ORDER = ("visible", "occluded")
|
||
HORIZONTAL_LATERAL_BIN_M = 5.0
|
||
HORIZONTAL_LATERAL_RANGE_M = 30.0
|
||
VERTICAL_DEPTH_BIN_M = 5.0
|
||
YAW_DEPTH_BIN_M = 10.0
|
||
YAW_HEADING_BIN_DEG = 10.0
|
||
ERROR_DISTANCE_BIN_M = 0.5
|
||
ERROR_YAW_BIN_DEG = 1.0
|
||
DEFAULT_ERROR_BIN_BADCASES = 50
|
||
DEFAULT_ERROR_BIN_SAMPLES_PER_BIN = 10
|
||
ROI1_MIN_Z_ERROR_DEPTH_M = 10.0
|
||
DEFAULT_YAW_COMPARE_MAX_LATERAL_DIST_M = HORIZONTAL_LATERAL_RANGE_M
|
||
DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M = 50.0
|
||
FACE_VISIBILITY_BUCKET_ORDER = ("front_rear_only", "side only", "two-face")
|
||
FOCUSED_CONFUSION_MAX_ABS_LATERAL_M = 5.0
|
||
FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M = 80.0
|
||
FOCUSED_CONFUSION_REQUIRED_DIFFICULTY = 0
|
||
ROI0_FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M = 30.0
|
||
LARGE_VEHICLE_CLASS_IDS = frozenset({5, 6, 7})
|
||
LARGE_VEHICLE_CLASS_SCOPE_TEXT = "5=bus, 6=truck/tanker/large_truck/construction_vehicle, 7=special_vehicle"
|
||
PORTRAIT_HEADING_BIN_DEG = 10.0
|
||
PORTRAIT_LATERAL_BIN_M = 5.0
|
||
PORTRAIT_LONGITUDINAL_BIN_M = 10.0
|
||
DEFAULT_DATA_PORTRAIT_WORKERS = max(1, os.cpu_count() or 1)
|
||
DATA_PORTRAIT_CHUNK_SIZE = 256
|
||
MIN_CONFIDENCE_FOR_2D_THRESHOLD_SEARCH = 0.001
|
||
BADCASE_FIELDS = [
|
||
"sample_index",
|
||
"roi",
|
||
"frame_name",
|
||
"image_path",
|
||
"label_path",
|
||
"cls_id",
|
||
"cls_name",
|
||
"gt_index",
|
||
"pred_index",
|
||
"confidence",
|
||
"match_iou",
|
||
"bbox_diag_px",
|
||
"bbox_diag_bin",
|
||
"distance_bin",
|
||
"is_cut_object",
|
||
"position_eligible",
|
||
"position_error_basis",
|
||
"yaw_compare_eligible",
|
||
"visible_face_count",
|
||
"visible_faces",
|
||
"has_side_face_visible",
|
||
"yaw_abs_deg",
|
||
"direct_visible_yaw_abs_deg",
|
||
"edge_visible_yaw_abs_deg",
|
||
"direct_minus_edge_visible_yaw_abs_deg",
|
||
"direct_length_abs_err_m",
|
||
"edge_length_abs_err_m",
|
||
"direct_minus_edge_length_abs_err_m",
|
||
"gt_lateral_abs_m",
|
||
"x_abs_m",
|
||
"y_abs_m",
|
||
"z_abs_m",
|
||
"center_error_m",
|
||
"position_gt_x_m",
|
||
"position_pred_x_m",
|
||
"position_gt_y_m",
|
||
"position_pred_y_m",
|
||
"position_gt_z_m",
|
||
"position_pred_z_m",
|
||
"gt_x_m",
|
||
"pred_x_m",
|
||
"gt_y_m",
|
||
"pred_y_m",
|
||
"gt_z_m",
|
||
"pred_z_m",
|
||
"gt_depth_m",
|
||
"pred_depth_m",
|
||
"gt_yaw_deg",
|
||
"pred_yaw_deg",
|
||
"gt_visible_yaw_deg",
|
||
"pred_edge_yaw_deg",
|
||
"gt_face_selection_label",
|
||
"pred_face_selection_label",
|
||
"face_selection_correct",
|
||
"gt_bbox_xyxy",
|
||
"pred_bbox_xyxy",
|
||
]
|
||
BADCASE_2D_FIELDS = [
|
||
"sample_index",
|
||
"roi",
|
||
"frame_name",
|
||
"image_path",
|
||
"label_path",
|
||
"kind",
|
||
"cls_id",
|
||
"cls_name",
|
||
"gt_index",
|
||
"pred_index",
|
||
"confidence",
|
||
"max_iou_any",
|
||
"max_iou_same_class",
|
||
"bbox_xyxy",
|
||
]
|
||
FAKE_CLASS_BADCASE_FIELDS = [
|
||
"sample_index",
|
||
"roi",
|
||
"frame_name",
|
||
"image_path",
|
||
"label_path",
|
||
"kind",
|
||
"gt_index",
|
||
"pred_index",
|
||
"gt_cls_id",
|
||
"gt_cls_name",
|
||
"pred_cls_id",
|
||
"pred_cls_name",
|
||
"gt_fake_class_label",
|
||
"pred_fake_class_label",
|
||
"gt_occlusion_label",
|
||
"pred_occlusion_label",
|
||
"match_iou",
|
||
"confidence",
|
||
"gt_bbox_xyxy",
|
||
"pred_bbox_xyxy",
|
||
]
|
||
FACE_SELECTION_BADCASE_FIELDS = [
|
||
"sample_index",
|
||
"roi",
|
||
"frame_name",
|
||
"image_path",
|
||
"label_path",
|
||
"gt_index",
|
||
"pred_index",
|
||
"gt_cls_id",
|
||
"gt_cls_name",
|
||
"pred_cls_id",
|
||
"pred_cls_name",
|
||
"gt_face_selection_label",
|
||
"pred_face_selection_label",
|
||
"match_iou",
|
||
"confidence",
|
||
"gt_bbox_xyxy",
|
||
"pred_bbox_xyxy",
|
||
]
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description="Run validation-set error analysis for the two-ROI Detect3D models and save bad-case visualizations."
|
||
)
|
||
parser.add_argument(
|
||
"--data",
|
||
type=str,
|
||
default=str(ROOT / "ultralytics" / "cfg" / "datasets" / "mono3d_ground.yaml"),
|
||
help="Dataset YAML path used to resolve the validation split.",
|
||
)
|
||
parser.add_argument("--split", type=str, default="val", choices=("train", "val", "test"), help="Dataset split to analyze.")
|
||
parser.add_argument(
|
||
"--output-root",
|
||
type=str,
|
||
default=str(DEFAULT_OUTPUT_ROOT),
|
||
help="Directory where summaries, CSVs, and bad-case visualizations will be written.",
|
||
)
|
||
parser.add_argument(
|
||
"--analyze-rois",
|
||
nargs="+",
|
||
default=["roi0", "roi1"],
|
||
choices=("roi0", "roi1"),
|
||
help="ROI models to analyze.",
|
||
)
|
||
parser.add_argument("--max-samples", type=int, default=0, help="Optional cap on analyzed samples. 0 means full split.")
|
||
parser.add_argument(
|
||
"--sample-selection",
|
||
type=str,
|
||
default="head",
|
||
choices=("head", "random"),
|
||
help="How to choose the first --max-samples entries when a cap is set.",
|
||
)
|
||
parser.add_argument("--sample-random-seed", type=int, default=20260327, help="Random seed used when --sample-selection=random.")
|
||
parser.add_argument("--log-every", type=int, default=100, help="Progress log interval in samples.")
|
||
parser.add_argument("--topk-badcases", type=int, default=100, help="How many worst cases to save per category and ROI.")
|
||
parser.add_argument("--per-class-badcases", type=int, default=50, help="How many bad cases to save per class/category and ROI.")
|
||
parser.add_argument(
|
||
"--error-bin-badcases",
|
||
type=int,
|
||
default=DEFAULT_ERROR_BIN_BADCASES,
|
||
help="How many object-level candidates to keep per error bin before frame-level aggregation for report sections.",
|
||
)
|
||
parser.add_argument(
|
||
"--error-bin-samples-per-bin",
|
||
type=int,
|
||
default=DEFAULT_ERROR_BIN_SAMPLES_PER_BIN,
|
||
help="How many frame-level sample images to export per error bin in the report sections.",
|
||
)
|
||
parser.add_argument("--badcase-random-seed", type=int, default=20260326, help="Random seed for bad-case visualization sampling.")
|
||
parser.add_argument("--yaw-bad-threshold-deg", type=float, default=5.0, help="Threshold for yaw bad-case CSV export.")
|
||
parser.add_argument(
|
||
"--yaw-compare-max-lateral-dist",
|
||
type=float,
|
||
default=DEFAULT_YAW_COMPARE_MAX_LATERAL_DIST_M,
|
||
help="Signed lateral range limit for yaw comparison bins; compares paired samples within [-limit, limit) meters.",
|
||
)
|
||
parser.add_argument("--horizontal-bad-threshold-m", type=float, default=0.5, help="Threshold for bad horizontal X error.")
|
||
parser.add_argument("--vertical-bad-threshold-m", type=float, default=0.5, help="Threshold for bad vertical Y error.")
|
||
parser.add_argument("--face-visibility-score-thresh", type=float, default=0.05, help="Visible-face score threshold.")
|
||
parser.add_argument("--torch-threads", type=int, default=0, help="Optional torch CPU thread count override.")
|
||
parser.add_argument("--skip-data-portrait", action="store_true", help="Skip dataset portrait generation.")
|
||
parser.add_argument(
|
||
"--data-portrait-split",
|
||
type=str,
|
||
default="train",
|
||
choices=("train", "val", "test"),
|
||
help="Dataset split used for the frame/object portrait statistics.",
|
||
)
|
||
parser.add_argument(
|
||
"--data-portrait-max-samples",
|
||
type=int,
|
||
default=0,
|
||
help="Optional cap on portrait samples. 0 means the full portrait split.",
|
||
)
|
||
parser.add_argument(
|
||
"--data-portrait-sample-selection",
|
||
type=str,
|
||
default="head",
|
||
choices=("head", "random"),
|
||
help="How to choose the first --data-portrait-max-samples entries when a cap is set.",
|
||
)
|
||
parser.add_argument(
|
||
"--data-portrait-random-seed",
|
||
type=int,
|
||
default=20260330,
|
||
help="Random seed used when --data-portrait-sample-selection=random.",
|
||
)
|
||
parser.add_argument(
|
||
"--data-portrait-workers",
|
||
type=int,
|
||
default=0,
|
||
help="Worker threads used for portrait GT/calibration scanning. 0 uses all detected CPU threads.",
|
||
)
|
||
add_two_roi_inference_args(parser, include_output_dir=False)
|
||
return parser.parse_args()
|
||
|
||
|
||
def load_yaml(path: str | Path) -> dict[str, Any]:
|
||
with Path(path).open("r", encoding="utf-8") as file:
|
||
return yaml.safe_load(file) or {}
|
||
|
||
|
||
def resolve_data_path(data_yaml: Path, dataset_root: Optional[str], value: Any) -> Path:
|
||
if value is None:
|
||
raise ValueError(f"Missing split path in {data_yaml}")
|
||
path = Path(str(value))
|
||
if path.is_absolute():
|
||
return path.resolve()
|
||
candidates = []
|
||
if dataset_root:
|
||
candidates.append((Path(dataset_root) / path).resolve())
|
||
candidates.append((data_yaml.parent / path).resolve())
|
||
for candidate in candidates:
|
||
if candidate.exists():
|
||
return candidate
|
||
return candidates[0] if candidates else path.resolve()
|
||
|
||
|
||
def load_split_entries(
|
||
split_file: Path,
|
||
max_samples: int = 0,
|
||
sample_selection: str = "head",
|
||
sample_random_seed: int = 20260327,
|
||
) -> list[tuple[str, str]]:
|
||
entries: list[tuple[str, str]] = []
|
||
with split_file.open("r", encoding="utf-8") as file:
|
||
for line in file:
|
||
entry = line.strip()
|
||
if not entry or entry.lstrip().startswith("#"):
|
||
continue
|
||
if Path(entry).suffix.lower() != ".txt":
|
||
raise ValueError(f"Ground3D split entries must point to label .txt files, but got: {entry}")
|
||
entries.append((str(split_file.parent.resolve()), entry))
|
||
if not entries:
|
||
raise FileNotFoundError(f"No usable entries found in {split_file}")
|
||
if max_samples > 0 and len(entries) > max_samples:
|
||
if sample_selection == "random":
|
||
rng = random.Random(int(sample_random_seed))
|
||
indices = sorted(rng.sample(range(len(entries)), int(max_samples)))
|
||
entries = [entries[idx] for idx in indices]
|
||
else:
|
||
entries = entries[:max_samples]
|
||
return entries
|
||
|
||
|
||
_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
|
||
|
||
|
||
def label_rel_to_image_rel(rel_label: Path) -> Path:
|
||
parts = list(rel_label.parts)
|
||
if "labels" in parts:
|
||
parts[len(parts) - 1 - parts[::-1].index("labels")] = "images"
|
||
return Path(*parts).with_suffix(".png")
|
||
|
||
|
||
def resolve_image_path(image_path: Path) -> Path:
|
||
if image_path.is_file():
|
||
return image_path
|
||
stem = image_path.stem
|
||
parent = image_path.parent
|
||
for ext in _IMAGE_EXTENSIONS:
|
||
if ext == image_path.suffix.lower():
|
||
continue
|
||
candidate = parent / (stem + ext)
|
||
if candidate.is_file():
|
||
return candidate
|
||
return image_path
|
||
|
||
|
||
def label_rel_to_calib_rel(rel_label: Path) -> Path:
|
||
parts = list(rel_label.parts)
|
||
if "labels" not in parts:
|
||
raise ValueError(f"Expected a Ground3D label path containing `labels`, but got: {rel_label}")
|
||
parts[len(parts) - 1 - parts[::-1].index("labels")] = "calib"
|
||
return Path(*parts).with_suffix(".json")
|
||
|
||
|
||
def entry_to_label_file(entry: tuple[str, str]) -> Path:
|
||
list_root, rel_label = entry
|
||
return (Path(list_root) / rel_label).resolve()
|
||
|
||
|
||
def label_path_to_calib_file(label_path: Path) -> Path:
|
||
return label_rel_to_calib_rel(label_path).resolve()
|
||
|
||
|
||
def entry_to_image_file(entry: tuple[str, str], image_root: Path) -> Path:
|
||
rel_image = label_rel_to_image_rel(Path(entry[1]))
|
||
return resolve_image_path((image_root / rel_image).resolve())
|
||
|
||
|
||
def read_raw_calib_from_label(image_path: Path, label_path: Path) -> Optional[dict[str, Any]]:
|
||
calib_path = label_path_to_calib_file(label_path)
|
||
return read_calib_from_path(str(image_path), extra_calib_candidates=[calib_path])
|
||
|
||
|
||
def infer_class_name_map(class_map: dict[str, int], preferred_names: Optional[dict[int, str]] = None) -> dict[int, str]:
|
||
names = dict(preferred_names or {})
|
||
for raw_name, cls_id in class_map.items():
|
||
names.setdefault(int(cls_id), str(raw_name))
|
||
return {int(cls_id): str(names[cls_id]) for cls_id in sorted(names)}
|
||
|
||
|
||
def read_mapped_label_class_names(label_file: Path, class_map: dict[str, int]) -> list[str]:
|
||
if not label_file.is_file():
|
||
return []
|
||
names: list[str] = []
|
||
with label_file.open("r", encoding="utf-8") as file:
|
||
for line in file:
|
||
parts = line.split()
|
||
if not parts:
|
||
continue
|
||
cls_name = str(parts[0])
|
||
if cls_name not in class_map:
|
||
continue
|
||
names.append(cls_name)
|
||
return names
|
||
|
||
|
||
def parse_label_frame_metadata(label_path: Path) -> dict[str, Optional[str]]:
|
||
stem = label_path.stem
|
||
parts = stem.split("_")
|
||
timestamp_index = next((index for index, token in enumerate(parts) if len(token) == 14 and token.isdigit()), None)
|
||
date_index = next((index for index, token in enumerate(parts) if len(token) == 8 and token.isdigit() and token.startswith("20")), None)
|
||
|
||
vehicle_alias_from_path = next((part for part in label_path.parts[:-1] if str(part).startswith("G1M3_")), None)
|
||
if vehicle_alias_from_path:
|
||
vehicle_alias = str(vehicle_alias_from_path)
|
||
elif timestamp_index is not None and timestamp_index > 0:
|
||
vehicle_alias = "_".join(parts[:timestamp_index])
|
||
elif date_index is not None and date_index > 0:
|
||
vehicle_alias = "_".join(parts[:date_index])
|
||
else:
|
||
vehicle_alias = "unknown"
|
||
timestamp_text = parts[timestamp_index] if timestamp_index is not None else None
|
||
|
||
frame_dt = None
|
||
if timestamp_text:
|
||
try:
|
||
frame_dt = datetime.strptime(timestamp_text, "%Y%m%d%H%M%S")
|
||
except ValueError:
|
||
frame_dt = None
|
||
if frame_dt is None and date_index is not None:
|
||
date_text = parts[date_index]
|
||
try:
|
||
frame_dt = datetime.strptime(date_text, "%Y%m%d")
|
||
except ValueError:
|
||
frame_dt = None
|
||
|
||
if frame_dt is None:
|
||
fallback_day = next((part for part in label_path.parts if len(part) == 8 and part.isdigit()), None)
|
||
if fallback_day:
|
||
try:
|
||
frame_dt = datetime.strptime(fallback_day, "%Y%m%d")
|
||
except ValueError:
|
||
frame_dt = None
|
||
|
||
return {
|
||
"frame_name": stem,
|
||
"vehicle_alias": vehicle_alias or "unknown",
|
||
"timestamp_text": timestamp_text,
|
||
"day": frame_dt.strftime("%Y-%m-%d") if frame_dt else None,
|
||
"hour": frame_dt.strftime("%H") if frame_dt else None,
|
||
}
|
||
|
||
|
||
def build_day_axis(day_keys: Iterable[str]) -> list[str]:
|
||
parsed_days = []
|
||
for key in {str(value) for value in day_keys if value}:
|
||
try:
|
||
parsed_days.append(datetime.strptime(key, "%Y-%m-%d"))
|
||
except ValueError:
|
||
continue
|
||
|
||
if not parsed_days:
|
||
return sorted({str(value) for value in day_keys if value})
|
||
|
||
start_day = min(parsed_days)
|
||
end_day = max(parsed_days)
|
||
axis = []
|
||
current = start_day
|
||
while current <= end_day:
|
||
axis.append(current.strftime("%Y-%m-%d"))
|
||
current += timedelta(days=1)
|
||
|
||
extras = sorted({str(value) for value in day_keys if value} - set(axis))
|
||
return axis + extras
|
||
|
||
|
||
def build_hour_axis(hour_keys: Iterable[str]) -> list[str]:
|
||
axis = [f"{hour:02d}" for hour in range(24)]
|
||
extras = sorted({str(value) for value in hour_keys if value} - set(axis))
|
||
return axis + extras
|
||
|
||
|
||
def build_counter_rows(counter: Counter[str], axis: list[str], key_name: str, count_name: str) -> list[dict[str, Any]]:
|
||
rows = []
|
||
seen = set()
|
||
for label in axis:
|
||
rows.append({key_name: label, count_name: int(counter.get(label, 0))})
|
||
seen.add(label)
|
||
for label, value in sorted(counter.items()):
|
||
if label in seen:
|
||
continue
|
||
rows.append({key_name: str(label), count_name: int(value)})
|
||
return rows
|
||
|
||
|
||
def wrap_heading_deg(angle_deg: float) -> float:
|
||
wrapped = ((float(angle_deg) + 180.0) % 360.0) - 180.0
|
||
return -180.0 if math.isclose(wrapped, 180.0, rel_tol=0.0, abs_tol=1e-9) else float(wrapped)
|
||
|
||
|
||
def heading_interval_start(angle_deg: Optional[float], bin_width_deg: float = PORTRAIT_HEADING_BIN_DEG) -> Optional[float]:
|
||
if angle_deg is None or not math.isfinite(angle_deg) or bin_width_deg <= 0:
|
||
return None
|
||
heading = wrap_heading_deg(float(angle_deg))
|
||
return math.floor((heading + 180.0) / bin_width_deg) * bin_width_deg - 180.0
|
||
|
||
|
||
def build_heading_rows(counter: Counter[float], bin_width_deg: float = PORTRAIT_HEADING_BIN_DEG) -> list[dict[str, Any]]:
|
||
if bin_width_deg <= 0:
|
||
return []
|
||
bin_count = max(1, int(round(360.0 / bin_width_deg)))
|
||
rows = []
|
||
for index in range(bin_count):
|
||
start_deg = -180.0 + index * bin_width_deg
|
||
rows.append(
|
||
{
|
||
"heading_bin_start_deg": float(start_deg),
|
||
"heading_bin_end_deg": float(start_deg + bin_width_deg),
|
||
"heading_bin_label": interval_label(float(start_deg), float(bin_width_deg), unit="deg"),
|
||
"count": int(counter.get(float(start_deg), 0)),
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def build_nonnegative_interval_rows(counter: Counter[float], bin_width_m: float, prefix: str) -> list[dict[str, Any]]:
|
||
if bin_width_m <= 0:
|
||
return []
|
||
max_start = max((float(start) for start, count in counter.items() if int(count) > 0), default=0.0)
|
||
num_bins = max(1, int(math.floor(max_start / bin_width_m)) + 1)
|
||
rows = []
|
||
for index in range(num_bins):
|
||
start_m = float(index * bin_width_m)
|
||
rows.append(
|
||
{
|
||
f"{prefix}_start_m": start_m,
|
||
f"{prefix}_end_m": float(start_m + bin_width_m),
|
||
f"{prefix}_label": interval_label(start_m, float(bin_width_m)),
|
||
"count": int(counter.get(start_m, 0)),
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def build_class_count_rows(
|
||
frame_counter: Counter[int],
|
||
object_counter: Counter[int],
|
||
class_names: dict[int, str],
|
||
class_order: list[int],
|
||
include_zero_rows: bool,
|
||
) -> list[dict[str, Any]]:
|
||
rows = []
|
||
for cls_id in class_order:
|
||
row = {
|
||
"cls_id": int(cls_id),
|
||
"cls_name": get_cls_name(class_names, int(cls_id)),
|
||
"frame_count": int(frame_counter.get(int(cls_id), 0)),
|
||
"object_count": int(object_counter.get(int(cls_id), 0)),
|
||
}
|
||
if include_zero_rows or row["frame_count"] > 0 or row["object_count"] > 0:
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def build_full_image_plot_calib(raw_calib: Optional[dict[str, Any]], ori_w: int, ori_h: int) -> Optional[dict[str, Any]]:
|
||
if raw_calib is None:
|
||
return None
|
||
fx = float(raw_calib.get("fx", raw_calib.get("focal_u", ori_w)))
|
||
fy = float(raw_calib.get("fy", raw_calib.get("focal_v", ori_h)))
|
||
cx = float(raw_calib.get("cx", raw_calib.get("cu", ori_w / 2.0)))
|
||
cy = float(raw_calib.get("cy", raw_calib.get("cv", ori_h / 2.0)))
|
||
return {
|
||
"fx": fx,
|
||
"fy": fy,
|
||
"cx": cx,
|
||
"cy": cy,
|
||
"distort_coeffs": list(raw_calib.get("distort_coeffs", [])),
|
||
"depth_scale": float(raw_calib.get("depth_scale", 1.0)),
|
||
}
|
||
|
||
|
||
def create_data_portrait_accumulators() -> dict[str, Any]:
|
||
return {
|
||
"frame_counts_by_vehicle": Counter(),
|
||
"day_counts_by_vehicle": defaultdict(Counter),
|
||
"hour_counts_by_vehicle": defaultdict(Counter),
|
||
"class_frame_counts_by_vehicle": defaultdict(Counter),
|
||
"class_object_counts_by_vehicle": defaultdict(Counter),
|
||
"heading_counts_by_vehicle": defaultdict(Counter),
|
||
"lateral_counts_by_class": defaultdict(Counter),
|
||
"longitudinal_counts_by_class": defaultdict(Counter),
|
||
"total_day_counts": Counter(),
|
||
"total_hour_counts": Counter(),
|
||
"total_class_frame_counts": Counter(),
|
||
"total_class_object_counts": Counter(),
|
||
"total_heading_counts": Counter(),
|
||
"all_day_keys": set(),
|
||
"all_hour_keys": set(),
|
||
"frames_with_mapped_objects": 0,
|
||
"frames_with_valid_3d": 0,
|
||
"processed_samples": 0,
|
||
}
|
||
|
||
|
||
def merge_counter_maps(dst: defaultdict[Any, Counter], src: defaultdict[Any, Counter]) -> None:
|
||
for key, counter in src.items():
|
||
dst[key].update(counter)
|
||
|
||
|
||
def merge_data_portrait_accumulators(dst: dict[str, Any], src: dict[str, Any]) -> None:
|
||
dst["frame_counts_by_vehicle"].update(src["frame_counts_by_vehicle"])
|
||
merge_counter_maps(dst["day_counts_by_vehicle"], src["day_counts_by_vehicle"])
|
||
merge_counter_maps(dst["hour_counts_by_vehicle"], src["hour_counts_by_vehicle"])
|
||
merge_counter_maps(dst["class_frame_counts_by_vehicle"], src["class_frame_counts_by_vehicle"])
|
||
merge_counter_maps(dst["class_object_counts_by_vehicle"], src["class_object_counts_by_vehicle"])
|
||
merge_counter_maps(dst["heading_counts_by_vehicle"], src["heading_counts_by_vehicle"])
|
||
merge_counter_maps(dst["lateral_counts_by_class"], src["lateral_counts_by_class"])
|
||
merge_counter_maps(dst["longitudinal_counts_by_class"], src["longitudinal_counts_by_class"])
|
||
dst["total_day_counts"].update(src["total_day_counts"])
|
||
dst["total_hour_counts"].update(src["total_hour_counts"])
|
||
dst["total_class_frame_counts"].update(src["total_class_frame_counts"])
|
||
dst["total_class_object_counts"].update(src["total_class_object_counts"])
|
||
dst["total_heading_counts"].update(src["total_heading_counts"])
|
||
dst["all_day_keys"].update(src["all_day_keys"])
|
||
dst["all_hour_keys"].update(src["all_hour_keys"])
|
||
dst["frames_with_mapped_objects"] += int(src["frames_with_mapped_objects"])
|
||
dst["frames_with_valid_3d"] += int(src["frames_with_valid_3d"])
|
||
dst["processed_samples"] += int(src["processed_samples"])
|
||
|
||
|
||
def resolve_data_portrait_workers(requested_workers: int, num_entries: int) -> int:
|
||
if num_entries <= 1:
|
||
return 1
|
||
if requested_workers and requested_workers > 0:
|
||
return max(1, min(int(requested_workers), int(num_entries)))
|
||
return max(1, min(int(DEFAULT_DATA_PORTRAIT_WORKERS), int(num_entries)))
|
||
|
||
|
||
def iter_data_portrait_chunks(entries: list[tuple[str, str]], chunk_size: int) -> Iterable[list[tuple[str, str]]]:
|
||
step = max(1, int(chunk_size))
|
||
for index in range(0, len(entries), step):
|
||
yield entries[index : index + step]
|
||
|
||
|
||
def process_data_portrait_chunk(
|
||
entries: list[tuple[str, str]],
|
||
image_root: Path,
|
||
class_map: dict[str, int],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
img_w: int,
|
||
img_h: int,
|
||
face_visibility_score_thresh: float,
|
||
) -> dict[str, Any]:
|
||
chunk_stats = create_data_portrait_accumulators()
|
||
|
||
for entry in entries:
|
||
label_path = entry_to_label_file(entry)
|
||
image_path = entry_to_image_file(entry, image_root)
|
||
frame_meta = parse_label_frame_metadata(label_path)
|
||
vehicle_alias = str(frame_meta.get("vehicle_alias") or "unknown")
|
||
day_key = frame_meta.get("day")
|
||
hour_key = frame_meta.get("hour")
|
||
|
||
chunk_stats["frame_counts_by_vehicle"][vehicle_alias] += 1
|
||
if day_key:
|
||
chunk_stats["day_counts_by_vehicle"][vehicle_alias][day_key] += 1
|
||
chunk_stats["total_day_counts"][day_key] += 1
|
||
chunk_stats["all_day_keys"].add(day_key)
|
||
if hour_key:
|
||
chunk_stats["hour_counts_by_vehicle"][vehicle_alias][hour_key] += 1
|
||
chunk_stats["total_hour_counts"][hour_key] += 1
|
||
chunk_stats["all_hour_keys"].add(hour_key)
|
||
|
||
lb_2d, lb_3d = parse_ground_3d_label_file(
|
||
str(label_path),
|
||
class_map,
|
||
difficulty_weights,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
min_wh=0.0,
|
||
)
|
||
cls_ids = lb_2d["cls"].reshape(-1).astype(np.int32) if len(lb_2d["cls"]) else np.zeros((0,), dtype=np.int32)
|
||
frame_cls_ids = {int(cls_id) for cls_id in cls_ids.tolist()}
|
||
if frame_cls_ids:
|
||
chunk_stats["frames_with_mapped_objects"] += 1
|
||
|
||
for cls_id in cls_ids.tolist():
|
||
cls_id_int = int(cls_id)
|
||
chunk_stats["class_object_counts_by_vehicle"][vehicle_alias][cls_id_int] += 1
|
||
chunk_stats["total_class_object_counts"][cls_id_int] += 1
|
||
for cls_id in frame_cls_ids:
|
||
chunk_stats["class_frame_counts_by_vehicle"][vehicle_alias][cls_id] += 1
|
||
chunk_stats["total_class_frame_counts"][cls_id] += 1
|
||
|
||
frame_has_valid_3d = False
|
||
if lb_3d is not None and len(lb_3d) and len(cls_ids):
|
||
raw_calib = read_raw_calib_from_label(image_path, label_path)
|
||
gt_calib = build_full_image_plot_calib(raw_calib, img_w, img_h)
|
||
if gt_calib is not None:
|
||
for gt_index, cls_id in enumerate(cls_ids.tolist()):
|
||
if gt_index >= len(lb_3d):
|
||
break
|
||
gt_attrs = extract_3d_attrs_from_gt(
|
||
lb_3d[gt_index],
|
||
int(cls_id),
|
||
gt_calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
score_thr=face_visibility_score_thresh,
|
||
)
|
||
if gt_attrs is None:
|
||
continue
|
||
|
||
frame_has_valid_3d = True
|
||
yaw_rad = to_float(gt_attrs.get("yaw"))
|
||
if yaw_rad is not None:
|
||
heading_start = heading_interval_start(math.degrees(yaw_rad), PORTRAIT_HEADING_BIN_DEG)
|
||
if heading_start is not None:
|
||
chunk_stats["heading_counts_by_vehicle"][vehicle_alias][heading_start] += 1
|
||
chunk_stats["total_heading_counts"][heading_start] += 1
|
||
|
||
center = gt_attrs.get("center")
|
||
lateral_m = None
|
||
longitudinal_m = None
|
||
if center is not None and len(center) >= 3:
|
||
lateral_m = to_float(center[0])
|
||
longitudinal_m = to_float(center[2])
|
||
|
||
if lateral_m is not None:
|
||
lateral_start = depth_interval_start(abs(float(lateral_m)), PORTRAIT_LATERAL_BIN_M)
|
||
if lateral_start is not None:
|
||
chunk_stats["lateral_counts_by_class"][int(cls_id)][float(lateral_start)] += 1
|
||
|
||
if longitudinal_m is not None and longitudinal_m >= 0.0:
|
||
longitudinal_start = depth_interval_start(float(longitudinal_m), PORTRAIT_LONGITUDINAL_BIN_M)
|
||
if longitudinal_start is not None:
|
||
chunk_stats["longitudinal_counts_by_class"][int(cls_id)][float(longitudinal_start)] += 1
|
||
|
||
if frame_has_valid_3d:
|
||
chunk_stats["frames_with_valid_3d"] += 1
|
||
|
||
chunk_stats["processed_samples"] = len(entries)
|
||
return chunk_stats
|
||
|
||
|
||
def maybe_log_data_portrait_progress(
|
||
split_name: str,
|
||
processed_samples: int,
|
||
total_samples: int,
|
||
frame_counts_by_vehicle: Counter[str],
|
||
total_class_object_counts: Counter[int],
|
||
start_time: float,
|
||
) -> None:
|
||
elapsed_minutes = (time.time() - start_time) / 60.0
|
||
print(
|
||
f"[portrait:{split_name}] {processed_samples}/{total_samples} samples "
|
||
f"vehicles={len(frame_counts_by_vehicle)} mapped_objects={sum(total_class_object_counts.values())} "
|
||
f"elapsed={elapsed_minutes:.2f}m"
|
||
)
|
||
|
||
|
||
def build_data_portrait(
|
||
entries: list[tuple[str, str]],
|
||
split_name: str,
|
||
split_path: Path,
|
||
image_root: Path,
|
||
output_root: Path,
|
||
class_map: dict[str, int],
|
||
class_names: dict[int, str],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
ori_img_size: tuple[int, int],
|
||
face_visibility_score_thresh: float,
|
||
log_every: int,
|
||
workers: int = 0,
|
||
) -> dict[str, Any]:
|
||
portrait_root = output_root / f"{str(split_name).lower()}_portrait"
|
||
portrait_root.mkdir(parents=True, exist_ok=True)
|
||
|
||
img_w = int(ori_img_size[0]) if ori_img_size and len(ori_img_size) > 0 else 1920
|
||
img_h = int(ori_img_size[1]) if ori_img_size and len(ori_img_size) > 1 else 1080
|
||
|
||
portrait_stats = create_data_portrait_accumulators()
|
||
start_time = time.time()
|
||
worker_count = resolve_data_portrait_workers(workers, len(entries))
|
||
next_log_sample = int(log_every) if log_every > 0 else None
|
||
chunks = list(iter_data_portrait_chunks(entries, DATA_PORTRAIT_CHUNK_SIZE))
|
||
|
||
if worker_count > 1 and len(chunks) > 1:
|
||
print(
|
||
f"[portrait:{split_name}] scanning GT/calib with {worker_count} worker threads "
|
||
f"(chunk_size={DATA_PORTRAIT_CHUNK_SIZE})..."
|
||
)
|
||
with ThreadPoolExecutor(max_workers=worker_count) as executor:
|
||
futures = [
|
||
executor.submit(
|
||
process_data_portrait_chunk,
|
||
entries=chunk,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
img_w=img_w,
|
||
img_h=img_h,
|
||
face_visibility_score_thresh=face_visibility_score_thresh,
|
||
)
|
||
for chunk in chunks
|
||
]
|
||
for future in as_completed(futures):
|
||
chunk_stats = future.result()
|
||
merge_data_portrait_accumulators(portrait_stats, chunk_stats)
|
||
while next_log_sample is not None and portrait_stats["processed_samples"] >= next_log_sample:
|
||
maybe_log_data_portrait_progress(
|
||
split_name,
|
||
next_log_sample,
|
||
len(entries),
|
||
portrait_stats["frame_counts_by_vehicle"],
|
||
portrait_stats["total_class_object_counts"],
|
||
start_time,
|
||
)
|
||
next_log_sample += int(log_every)
|
||
else:
|
||
for chunk in chunks:
|
||
chunk_stats = process_data_portrait_chunk(
|
||
entries=chunk,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
img_w=img_w,
|
||
img_h=img_h,
|
||
face_visibility_score_thresh=face_visibility_score_thresh,
|
||
)
|
||
merge_data_portrait_accumulators(portrait_stats, chunk_stats)
|
||
while next_log_sample is not None and portrait_stats["processed_samples"] >= next_log_sample:
|
||
maybe_log_data_portrait_progress(
|
||
split_name,
|
||
next_log_sample,
|
||
len(entries),
|
||
portrait_stats["frame_counts_by_vehicle"],
|
||
portrait_stats["total_class_object_counts"],
|
||
start_time,
|
||
)
|
||
next_log_sample += int(log_every)
|
||
|
||
if log_every > 0:
|
||
last_logged_sample = max(0, (next_log_sample or int(log_every)) - int(log_every))
|
||
else:
|
||
last_logged_sample = 0
|
||
if log_every > 0 and portrait_stats["processed_samples"] > 0 and last_logged_sample < len(entries):
|
||
maybe_log_data_portrait_progress(
|
||
split_name,
|
||
len(entries),
|
||
len(entries),
|
||
portrait_stats["frame_counts_by_vehicle"],
|
||
portrait_stats["total_class_object_counts"],
|
||
start_time,
|
||
)
|
||
|
||
frame_counts_by_vehicle = portrait_stats["frame_counts_by_vehicle"]
|
||
day_counts_by_vehicle = portrait_stats["day_counts_by_vehicle"]
|
||
hour_counts_by_vehicle = portrait_stats["hour_counts_by_vehicle"]
|
||
class_frame_counts_by_vehicle = portrait_stats["class_frame_counts_by_vehicle"]
|
||
class_object_counts_by_vehicle = portrait_stats["class_object_counts_by_vehicle"]
|
||
heading_counts_by_vehicle = portrait_stats["heading_counts_by_vehicle"]
|
||
lateral_counts_by_class = portrait_stats["lateral_counts_by_class"]
|
||
longitudinal_counts_by_class = portrait_stats["longitudinal_counts_by_class"]
|
||
total_day_counts = portrait_stats["total_day_counts"]
|
||
total_hour_counts = portrait_stats["total_hour_counts"]
|
||
total_class_frame_counts = portrait_stats["total_class_frame_counts"]
|
||
total_class_object_counts = portrait_stats["total_class_object_counts"]
|
||
total_heading_counts = portrait_stats["total_heading_counts"]
|
||
all_day_keys = portrait_stats["all_day_keys"]
|
||
all_hour_keys = portrait_stats["all_hour_keys"]
|
||
frames_with_mapped_objects = portrait_stats["frames_with_mapped_objects"]
|
||
frames_with_valid_3d = portrait_stats["frames_with_valid_3d"]
|
||
|
||
vehicle_order = [alias for alias, _ in sorted(frame_counts_by_vehicle.items(), key=lambda item: (-int(item[1]), str(item[0])))]
|
||
class_order = sorted(class_names, key=lambda cls_id: (-int(total_class_object_counts.get(int(cls_id), 0)), int(cls_id)))
|
||
if not class_order:
|
||
class_order = sorted({int(cls_id) for cls_id in total_class_object_counts})
|
||
|
||
day_axis = build_day_axis(all_day_keys)
|
||
hour_axis = build_hour_axis(all_hour_keys)
|
||
|
||
vehicle_rows = [{"vehicle_alias": alias, "frame_count": int(frame_counts_by_vehicle[alias])} for alias in vehicle_order]
|
||
daily_total_rows = build_counter_rows(total_day_counts, day_axis, "day", "frame_count")
|
||
hourly_total_rows = build_counter_rows(total_hour_counts, hour_axis, "hour", "frame_count")
|
||
heading_total_rows = build_heading_rows(total_heading_counts, PORTRAIT_HEADING_BIN_DEG)
|
||
class_total_rows = build_class_count_rows(total_class_frame_counts, total_class_object_counts, class_names, class_order, include_zero_rows=False)
|
||
|
||
daily_cards = [
|
||
{
|
||
"title": "All Vehicles",
|
||
"subtitle": f"frames={len(entries)}",
|
||
"rows": daily_total_rows,
|
||
}
|
||
]
|
||
hourly_cards = [
|
||
{
|
||
"title": "All Vehicles",
|
||
"subtitle": f"frames={len(entries)}",
|
||
"rows": hourly_total_rows,
|
||
}
|
||
]
|
||
heading_cards = [
|
||
{
|
||
"title": "All Vehicles",
|
||
"subtitle": f"objects={sum(int(row.get('count', 0)) for row in heading_total_rows)}",
|
||
"rows": heading_total_rows,
|
||
}
|
||
]
|
||
class_vehicle_groups = []
|
||
for alias in vehicle_order:
|
||
alias_daily_rows = build_counter_rows(day_counts_by_vehicle[alias], day_axis, "day", "frame_count")
|
||
alias_hourly_rows = build_counter_rows(hour_counts_by_vehicle[alias], hour_axis, "hour", "frame_count")
|
||
alias_heading_rows = build_heading_rows(heading_counts_by_vehicle[alias], PORTRAIT_HEADING_BIN_DEG)
|
||
alias_class_rows = build_class_count_rows(
|
||
class_frame_counts_by_vehicle[alias],
|
||
class_object_counts_by_vehicle[alias],
|
||
class_names,
|
||
class_order,
|
||
include_zero_rows=False,
|
||
)
|
||
|
||
daily_cards.append({"title": alias, "subtitle": f"frames={frame_counts_by_vehicle[alias]}", "rows": alias_daily_rows})
|
||
hourly_cards.append({"title": alias, "subtitle": f"frames={frame_counts_by_vehicle[alias]}", "rows": alias_hourly_rows})
|
||
heading_cards.append(
|
||
{
|
||
"title": alias,
|
||
"subtitle": f"objects={sum(int(row.get('count', 0)) for row in alias_heading_rows)}",
|
||
"rows": alias_heading_rows,
|
||
}
|
||
)
|
||
class_vehicle_groups.append(
|
||
{
|
||
"vehicle_alias": alias,
|
||
"summary": (
|
||
f"{alias} "
|
||
f"(frames={frame_counts_by_vehicle[alias]}, "
|
||
f"class_frames={sum(int(row.get('frame_count', 0)) for row in alias_class_rows)}, "
|
||
f"objects={sum(int(row.get('object_count', 0)) for row in alias_class_rows)})"
|
||
),
|
||
"rows": alias_class_rows,
|
||
}
|
||
)
|
||
|
||
lateral_cards = []
|
||
longitudinal_cards = []
|
||
lateral_rows_flat = []
|
||
longitudinal_rows_flat = []
|
||
for cls_id in class_order:
|
||
cls_name = get_cls_name(class_names, int(cls_id))
|
||
lateral_rows = build_nonnegative_interval_rows(lateral_counts_by_class[int(cls_id)], PORTRAIT_LATERAL_BIN_M, "lateral_bin")
|
||
lateral_rows = [
|
||
{"cls_id": int(cls_id), "cls_name": cls_name, **row}
|
||
for row in lateral_rows
|
||
if int(row.get("count", 0)) > 0
|
||
]
|
||
if lateral_rows:
|
||
lateral_cards.append(
|
||
{
|
||
"title": cls_name,
|
||
"subtitle": f"id={int(cls_id)} objects={sum(int(row.get('count', 0)) for row in lateral_rows)}",
|
||
"rows": lateral_rows,
|
||
}
|
||
)
|
||
lateral_rows_flat.extend(lateral_rows)
|
||
|
||
longitudinal_rows = build_nonnegative_interval_rows(
|
||
longitudinal_counts_by_class[int(cls_id)], PORTRAIT_LONGITUDINAL_BIN_M, "longitudinal_bin"
|
||
)
|
||
longitudinal_rows = [
|
||
{"cls_id": int(cls_id), "cls_name": cls_name, **row}
|
||
for row in longitudinal_rows
|
||
if int(row.get("count", 0)) > 0
|
||
]
|
||
if longitudinal_rows:
|
||
longitudinal_cards.append(
|
||
{
|
||
"title": cls_name,
|
||
"subtitle": f"id={int(cls_id)} objects={sum(int(row.get('count', 0)) for row in longitudinal_rows)}",
|
||
"rows": longitudinal_rows,
|
||
}
|
||
)
|
||
longitudinal_rows_flat.extend(longitudinal_rows)
|
||
|
||
frames_by_vehicle_csv = portrait_root / "frames_by_vehicle.csv"
|
||
frames_by_day_csv = portrait_root / "frames_by_day.csv"
|
||
frames_by_hour_csv = portrait_root / "frames_by_hour.csv"
|
||
class_counts_csv = portrait_root / "class_counts.csv"
|
||
heading_counts_csv = portrait_root / "heading_counts.csv"
|
||
lateral_counts_csv = portrait_root / "class_lateral_distance_5m.csv"
|
||
longitudinal_counts_csv = portrait_root / "class_longitudinal_distance_10m.csv"
|
||
|
||
write_rows_csv(frames_by_vehicle_csv, vehicle_rows, fallback_fields=["vehicle_alias", "frame_count"])
|
||
write_rows_csv(
|
||
frames_by_day_csv,
|
||
[{"scope": "total", "vehicle_alias": "all", **row} for row in daily_total_rows]
|
||
+ [
|
||
{"scope": "vehicle", "vehicle_alias": card["title"], **row}
|
||
for card in daily_cards[1:]
|
||
for row in card["rows"]
|
||
],
|
||
fallback_fields=["scope", "vehicle_alias", "day", "frame_count"],
|
||
)
|
||
write_rows_csv(
|
||
frames_by_hour_csv,
|
||
[{"scope": "total", "vehicle_alias": "all", **row} for row in hourly_total_rows]
|
||
+ [
|
||
{"scope": "vehicle", "vehicle_alias": card["title"], **row}
|
||
for card in hourly_cards[1:]
|
||
for row in card["rows"]
|
||
],
|
||
fallback_fields=["scope", "vehicle_alias", "hour", "frame_count"],
|
||
)
|
||
write_rows_csv(
|
||
class_counts_csv,
|
||
[{"scope": "total", "vehicle_alias": "all", **row} for row in class_total_rows]
|
||
+ [
|
||
{"scope": "vehicle", "vehicle_alias": str(group.get("vehicle_alias", "unknown")), **row}
|
||
for group in class_vehicle_groups
|
||
for row in group["rows"]
|
||
],
|
||
fallback_fields=["scope", "vehicle_alias", "cls_id", "cls_name", "frame_count", "object_count"],
|
||
)
|
||
write_rows_csv(
|
||
heading_counts_csv,
|
||
[{"scope": "total", "vehicle_alias": "all", **row} for row in heading_total_rows]
|
||
+ [
|
||
{"scope": "vehicle", "vehicle_alias": card["title"], **row}
|
||
for card in heading_cards[1:]
|
||
for row in card["rows"]
|
||
],
|
||
fallback_fields=["scope", "vehicle_alias", "heading_bin_start_deg", "heading_bin_end_deg", "heading_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
lateral_counts_csv,
|
||
lateral_rows_flat,
|
||
fallback_fields=["cls_id", "cls_name", "lateral_bin_start_m", "lateral_bin_end_m", "lateral_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
longitudinal_counts_csv,
|
||
longitudinal_rows_flat,
|
||
fallback_fields=[
|
||
"cls_id",
|
||
"cls_name",
|
||
"longitudinal_bin_start_m",
|
||
"longitudinal_bin_end_m",
|
||
"longitudinal_bin_label",
|
||
"count",
|
||
],
|
||
)
|
||
|
||
summary_payload = {
|
||
"split": str(split_name),
|
||
"split_path": str(split_path),
|
||
"image_root": str(image_root),
|
||
"artifact_root": str(portrait_root),
|
||
"summary": {
|
||
"num_entries": len(entries),
|
||
"vehicles": len(vehicle_order),
|
||
"frames_with_mapped_objects": int(frames_with_mapped_objects),
|
||
"frames_with_valid_3d": int(frames_with_valid_3d),
|
||
"mapped_objects": int(sum(total_class_object_counts.values())),
|
||
"heading_objects": int(sum(total_heading_counts.values())),
|
||
"lateral_objects": int(sum(int(row.get("count", 0)) for row in lateral_rows_flat)),
|
||
"longitudinal_objects": int(sum(int(row.get("count", 0)) for row in longitudinal_rows_flat)),
|
||
},
|
||
"vehicle_rows": vehicle_rows,
|
||
"daily_cards": daily_cards,
|
||
"hourly_cards": hourly_cards,
|
||
"class_total_rows": class_total_rows,
|
||
"class_vehicle_groups": class_vehicle_groups,
|
||
"heading_cards": heading_cards,
|
||
"lateral_cards": lateral_cards,
|
||
"longitudinal_cards": longitudinal_cards,
|
||
"artifact_paths": {
|
||
"frames_by_vehicle_csv": str(frames_by_vehicle_csv),
|
||
"frames_by_day_csv": str(frames_by_day_csv),
|
||
"frames_by_hour_csv": str(frames_by_hour_csv),
|
||
"class_counts_csv": str(class_counts_csv),
|
||
"heading_counts_csv": str(heading_counts_csv),
|
||
"lateral_counts_csv": str(lateral_counts_csv),
|
||
"longitudinal_counts_csv": str(longitudinal_counts_csv),
|
||
},
|
||
}
|
||
summary_path = portrait_root / "summary.json"
|
||
write_json(summary_path, summary_payload)
|
||
summary_payload["summary_path"] = str(summary_path)
|
||
return summary_payload
|
||
|
||
|
||
def load_existing_data_portrait(output_root: Path, split_name: str) -> Optional[dict[str, Any]]:
|
||
portrait_root = output_root / f"{str(split_name).lower()}_portrait"
|
||
summary_path = portrait_root / "summary.json"
|
||
if not summary_path.is_file():
|
||
return None
|
||
payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||
if not isinstance(payload, dict):
|
||
return None
|
||
payload = dict(payload)
|
||
payload["summary_path"] = str(summary_path)
|
||
return payload
|
||
|
||
|
||
def filter_labels_by_classes(
|
||
lb_2d: dict[str, Any], lb_3d: Optional[np.ndarray], include_classes: Optional[set[int]]
|
||
) -> tuple[dict[str, Any], Optional[np.ndarray]]:
|
||
if include_classes is None or len(lb_2d["cls"]) == 0:
|
||
return lb_2d, lb_3d
|
||
cls = lb_2d["cls"].reshape(-1).astype(np.int32)
|
||
keep = np.isin(cls, np.asarray(sorted(include_classes), dtype=np.int32))
|
||
filtered_2d = {
|
||
**lb_2d,
|
||
"cls": lb_2d["cls"][keep],
|
||
"bboxes": lb_2d["bboxes"][keep],
|
||
"difficulties": lb_2d["difficulties"][keep],
|
||
}
|
||
if lb_3d is None:
|
||
return filtered_2d, None
|
||
return filtered_2d, lb_3d[keep]
|
||
|
||
|
||
def filter_label_names_by_classes(class_names: list[str], cls_ids: np.ndarray, include_classes: Optional[set[int]]) -> list[str]:
|
||
if include_classes is None or not class_names:
|
||
return class_names
|
||
cls = np.asarray(cls_ids, dtype=np.int32).reshape(-1)
|
||
keep = np.isin(cls, np.asarray(sorted(include_classes), dtype=np.int32))
|
||
return [name for name, keep_item in zip(class_names, keep.tolist()) if keep_item]
|
||
|
||
|
||
def filter_labels_by_min_wh(
|
||
lb_2d: dict[str, Any],
|
||
lb_3d: Optional[np.ndarray],
|
||
img_w: int,
|
||
img_h: int,
|
||
min_wh_px: float,
|
||
) -> tuple[dict[str, Any], Optional[np.ndarray]]:
|
||
if min_wh_px <= 0 or len(lb_2d["bboxes"]) == 0:
|
||
return lb_2d, lb_3d
|
||
bboxes = np.asarray(lb_2d["bboxes"], dtype=np.float32)
|
||
w_px = bboxes[:, 2] * float(img_w)
|
||
h_px = bboxes[:, 3] * float(img_h)
|
||
keep = (w_px >= float(min_wh_px)) | (h_px >= float(min_wh_px))
|
||
if keep.all():
|
||
return lb_2d, lb_3d
|
||
filtered_2d = {
|
||
**lb_2d,
|
||
"cls": lb_2d["cls"][keep],
|
||
"bboxes": lb_2d["bboxes"][keep],
|
||
"difficulties": lb_2d["difficulties"][keep],
|
||
}
|
||
if lb_3d is None:
|
||
return filtered_2d, None
|
||
return filtered_2d, lb_3d[keep]
|
||
|
||
|
||
def filter_label_names_by_min_wh(class_names: list[str], bboxes: np.ndarray, img_w: int, img_h: int, min_wh_px: float) -> list[str]:
|
||
if min_wh_px <= 0 or not class_names:
|
||
return class_names
|
||
boxes = np.asarray(bboxes, dtype=np.float32)
|
||
if boxes.size == 0:
|
||
return []
|
||
w_px = boxes[:, 2] * float(img_w)
|
||
h_px = boxes[:, 3] * float(img_h)
|
||
keep = (w_px >= float(min_wh_px)) | (h_px >= float(min_wh_px))
|
||
return [name for name, keep_item in zip(class_names, keep.tolist()) if keep_item]
|
||
|
||
|
||
def xywhn_to_xyxy(boxes_xywhn: np.ndarray, img_w: int, img_h: int) -> np.ndarray:
|
||
if boxes_xywhn is None or len(boxes_xywhn) == 0:
|
||
return np.zeros((0, 4), dtype=np.float32)
|
||
boxes = np.asarray(boxes_xywhn, dtype=np.float32).copy()
|
||
boxes[:, 0] = (boxes_xywhn[:, 0] - boxes_xywhn[:, 2] * 0.5) * img_w
|
||
boxes[:, 1] = (boxes_xywhn[:, 1] - boxes_xywhn[:, 3] * 0.5) * img_h
|
||
boxes[:, 2] = (boxes_xywhn[:, 0] + boxes_xywhn[:, 2] * 0.5) * img_w
|
||
boxes[:, 3] = (boxes_xywhn[:, 1] + boxes_xywhn[:, 3] * 0.5) * img_h
|
||
return boxes
|
||
|
||
|
||
def box_iou_matrix(gt_boxes: np.ndarray, pred_boxes: np.ndarray) -> np.ndarray:
|
||
if len(gt_boxes) == 0 or len(pred_boxes) == 0:
|
||
return np.zeros((len(gt_boxes), len(pred_boxes)), dtype=np.float32)
|
||
gt = gt_boxes.astype(np.float32)
|
||
pred = pred_boxes.astype(np.float32)
|
||
inter_x1 = np.maximum(gt[:, None, 0], pred[None, :, 0])
|
||
inter_y1 = np.maximum(gt[:, None, 1], pred[None, :, 1])
|
||
inter_x2 = np.minimum(gt[:, None, 2], pred[None, :, 2])
|
||
inter_y2 = np.minimum(gt[:, None, 3], pred[None, :, 3])
|
||
inter_w = np.clip(inter_x2 - inter_x1, 0.0, None)
|
||
inter_h = np.clip(inter_y2 - inter_y1, 0.0, None)
|
||
inter = inter_w * inter_h
|
||
gt_area = np.clip(gt[:, 2] - gt[:, 0], 0.0, None) * np.clip(gt[:, 3] - gt[:, 1], 0.0, None)
|
||
pred_area = np.clip(pred[:, 2] - pred[:, 0], 0.0, None) * np.clip(pred[:, 3] - pred[:, 1], 0.0, None)
|
||
union = gt_area[:, None] + pred_area[None, :] - inter
|
||
return inter / np.clip(union, 1e-6, None)
|
||
|
||
|
||
def greedy_match_indices(gt_cls: np.ndarray, gt_boxes: np.ndarray, pred_cls: np.ndarray, pred_boxes: np.ndarray, iou_thr: float = 0.5):
|
||
if len(gt_boxes) == 0 or len(pred_boxes) == 0:
|
||
return np.zeros((0, 2), dtype=np.int32), np.zeros((len(gt_boxes), len(pred_boxes)), dtype=np.float32)
|
||
iou = box_iou_matrix(gt_boxes, pred_boxes)
|
||
class_mask = gt_cls[:, None] == pred_cls[None, :]
|
||
iou = iou * class_mask.astype(np.float32)
|
||
matches = np.array(np.nonzero(iou >= iou_thr)).T
|
||
if matches.shape[0] == 0:
|
||
return matches.astype(np.int32), iou
|
||
if matches.shape[0] > 1:
|
||
scores = iou[matches[:, 0], matches[:, 1]]
|
||
matches = matches[scores.argsort()[::-1]]
|
||
matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
|
||
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
|
||
return matches.astype(np.int32), iou
|
||
|
||
|
||
def greedy_match_indices_any_class(gt_boxes: np.ndarray, pred_boxes: np.ndarray, iou_thr: float = 0.5):
|
||
if len(gt_boxes) == 0 or len(pred_boxes) == 0:
|
||
return np.zeros((0, 2), dtype=np.int32), np.zeros((len(gt_boxes), len(pred_boxes)), dtype=np.float32)
|
||
iou = box_iou_matrix(gt_boxes, pred_boxes)
|
||
matches = np.array(np.nonzero(iou >= iou_thr)).T
|
||
if matches.shape[0] == 0:
|
||
return matches.astype(np.int32), iou
|
||
if matches.shape[0] > 1:
|
||
scores = iou[matches[:, 0], matches[:, 1]]
|
||
matches = matches[scores.argsort()[::-1]]
|
||
matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
|
||
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
|
||
return matches.astype(np.int32), iou
|
||
|
||
|
||
def is_focused_confusion_spatial_object(
|
||
lateral_m: Optional[float],
|
||
longitudinal_m: Optional[float],
|
||
max_abs_lateral_m: float = FOCUSED_CONFUSION_MAX_ABS_LATERAL_M,
|
||
max_abs_longitudinal_m: float = FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M,
|
||
) -> bool:
|
||
if lateral_m is None or longitudinal_m is None:
|
||
return False
|
||
return abs(float(lateral_m)) < float(max_abs_lateral_m) and abs(float(longitudinal_m)) < float(max_abs_longitudinal_m)
|
||
|
||
|
||
def is_focused_confusion_gt_object(
|
||
lateral_m: Optional[float],
|
||
longitudinal_m: Optional[float],
|
||
difficulty: Any,
|
||
max_abs_lateral_m: float = FOCUSED_CONFUSION_MAX_ABS_LATERAL_M,
|
||
max_abs_longitudinal_m: float = FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M,
|
||
required_difficulty: int = FOCUSED_CONFUSION_REQUIRED_DIFFICULTY,
|
||
) -> bool:
|
||
difficulty_value = to_float(difficulty)
|
||
if difficulty_value is None or int(round(float(difficulty_value))) != int(required_difficulty):
|
||
return False
|
||
return is_focused_confusion_spatial_object(
|
||
lateral_m=lateral_m,
|
||
longitudinal_m=longitudinal_m,
|
||
max_abs_lateral_m=max_abs_lateral_m,
|
||
max_abs_longitudinal_m=max_abs_longitudinal_m,
|
||
)
|
||
|
||
|
||
def focused_confusion_max_abs_longitudinal_m_for_roi(roi_name: str) -> float:
|
||
return float(ROI0_FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M if str(roi_name).lower() == "roi0" else FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M)
|
||
|
||
|
||
def resolve_face_center_eval_face_type(
|
||
gt_visible_faces: list[tuple[int, np.ndarray]],
|
||
gt_decoded: Optional[dict[str, Any]],
|
||
pred_decoded: Optional[dict[str, Any]],
|
||
) -> Optional[int]:
|
||
gt_visible_face_types = {int(face_type) for face_type, _ in gt_visible_faces}
|
||
pred_visible_face_type = None if pred_decoded is None else pred_decoded.get("visible_face_type")
|
||
if pred_visible_face_type is not None and int(pred_visible_face_type) in gt_visible_face_types:
|
||
return int(pred_visible_face_type)
|
||
gt_visible_face_type = None if gt_decoded is None else gt_decoded.get("visible_face_type")
|
||
if gt_visible_face_type is not None and int(gt_visible_face_type) in gt_visible_face_types:
|
||
return int(gt_visible_face_type)
|
||
return None
|
||
|
||
|
||
def update_subset_confusion_matrix(
|
||
confusion_matrix: ConfusionMatrix,
|
||
gt_cls: np.ndarray,
|
||
gt_boxes: np.ndarray,
|
||
pred_cls: np.ndarray,
|
||
pred_boxes: np.ndarray,
|
||
pred_conf: np.ndarray,
|
||
gt_mask: np.ndarray,
|
||
pred_mask: np.ndarray,
|
||
conf: float = 0.0,
|
||
iou_thres: float = 0.5,
|
||
) -> None:
|
||
gt_cls = np.asarray(gt_cls, dtype=np.int32).reshape(-1)
|
||
gt_boxes = np.asarray(gt_boxes, dtype=np.float32).reshape(-1, 4)
|
||
pred_cls = np.asarray(pred_cls, dtype=np.int32).reshape(-1)
|
||
pred_boxes = np.asarray(pred_boxes, dtype=np.float32).reshape(-1, 4)
|
||
pred_conf = np.asarray(pred_conf, dtype=np.float32).reshape(-1)
|
||
gt_mask = np.asarray(gt_mask, dtype=bool).reshape(-1)
|
||
pred_mask = np.asarray(pred_mask, dtype=bool).reshape(-1)
|
||
|
||
if gt_mask.shape[0] != gt_cls.shape[0] or pred_mask.shape[0] != pred_cls.shape[0]:
|
||
raise ValueError("Focused confusion matrix masks must align with GT/pred arrays.")
|
||
|
||
if pred_conf.size:
|
||
keep = pred_conf > float(conf)
|
||
pred_cls = pred_cls[keep]
|
||
pred_boxes = pred_boxes[keep]
|
||
pred_mask = pred_mask[keep]
|
||
else:
|
||
pred_cls = np.zeros((0,), dtype=np.int32)
|
||
pred_boxes = np.zeros((0, 4), dtype=np.float32)
|
||
pred_mask = np.zeros((0,), dtype=bool)
|
||
|
||
matches, _ = greedy_match_indices_any_class(gt_boxes, pred_boxes, iou_thr=iou_thres)
|
||
matched_gt_to_pred = {int(gt_index): int(pred_index) for gt_index, pred_index in matches.tolist()}
|
||
matched_pred = {int(pred_index) for _, pred_index in matches.tolist()}
|
||
|
||
for gt_index, gt_class in enumerate(gt_cls.tolist()):
|
||
if not bool(gt_mask[gt_index]):
|
||
continue
|
||
pred_index = matched_gt_to_pred.get(int(gt_index))
|
||
if pred_index is None:
|
||
confusion_matrix.matrix[confusion_matrix.nc, int(gt_class)] += 1
|
||
continue
|
||
confusion_matrix.matrix[int(pred_cls[pred_index]), int(gt_class)] += 1
|
||
|
||
for pred_index, pred_class in enumerate(pred_cls.tolist()):
|
||
if int(pred_index) in matched_pred or not bool(pred_mask[pred_index]):
|
||
continue
|
||
confusion_matrix.matrix[int(pred_class), confusion_matrix.nc] += 1
|
||
|
||
|
||
def compute_2d_tp_matrix(
|
||
gt_cls: np.ndarray,
|
||
gt_boxes: np.ndarray,
|
||
pred_cls: np.ndarray,
|
||
pred_boxes: np.ndarray,
|
||
iou_thresholds: np.ndarray,
|
||
) -> np.ndarray:
|
||
correct = np.zeros((len(pred_cls), len(iou_thresholds)), dtype=bool)
|
||
if len(gt_boxes) == 0 or len(pred_boxes) == 0:
|
||
return correct
|
||
iou = box_iou_matrix(gt_boxes, pred_boxes)
|
||
correct_class = gt_cls[:, None] == pred_cls[None, :]
|
||
iou = iou * correct_class.astype(np.float32)
|
||
for idx, threshold in enumerate(iou_thresholds.tolist()):
|
||
matches = np.array(np.nonzero(iou >= threshold)).T
|
||
if matches.shape[0]:
|
||
if matches.shape[0] > 1:
|
||
matches = matches[iou[matches[:, 0], matches[:, 1]].argsort()[::-1]]
|
||
matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
|
||
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
|
||
correct[matches[:, 1].astype(int), idx] = True
|
||
return correct
|
||
|
||
|
||
def make_2d_ap_store() -> dict[str, list[np.ndarray]]:
|
||
return {"tp": [], "conf": [], "pred_cls": [], "target_cls": []}
|
||
|
||
|
||
def add_2d_ap_sample(
|
||
store: dict[str, list[np.ndarray]],
|
||
gt_cls: np.ndarray,
|
||
gt_boxes: np.ndarray,
|
||
pred_cls: np.ndarray,
|
||
pred_boxes: np.ndarray,
|
||
pred_conf: np.ndarray,
|
||
iou_thresholds: np.ndarray,
|
||
) -> None:
|
||
store["target_cls"].append(np.asarray(gt_cls, dtype=np.int32))
|
||
if len(pred_cls) == 0:
|
||
return
|
||
tp = compute_2d_tp_matrix(
|
||
np.asarray(gt_cls, dtype=np.int32),
|
||
np.asarray(gt_boxes, dtype=np.float32),
|
||
np.asarray(pred_cls, dtype=np.int32),
|
||
np.asarray(pred_boxes, dtype=np.float32),
|
||
iou_thresholds=np.asarray(iou_thresholds, dtype=np.float32),
|
||
)
|
||
store["tp"].append(tp.astype(np.float32))
|
||
store["conf"].append(np.asarray(pred_conf, dtype=np.float32))
|
||
store["pred_cls"].append(np.asarray(pred_cls, dtype=np.int32))
|
||
|
||
|
||
def summarize_2d_ap_store(
|
||
store: dict[str, list[np.ndarray]],
|
||
plot: bool = False,
|
||
save_dir: Optional[Path] = None,
|
||
names: Optional[dict[int, str]] = None,
|
||
prefix: str = "",
|
||
) -> dict[str, Any]:
|
||
target_cls = np.concatenate(store["target_cls"], axis=0) if store["target_cls"] else np.zeros((0,), dtype=np.int32)
|
||
fallback_threshold = float(MIN_CONFIDENCE_FOR_2D_THRESHOLD_SEARCH)
|
||
if not store["tp"] or target_cls.size == 0:
|
||
unique_classes = np.unique(target_cls)
|
||
return {
|
||
"map50": None,
|
||
"map50_95": None,
|
||
"threshold_advice": {
|
||
"recommended_confidence": fallback_threshold,
|
||
"mean_precision": None,
|
||
"mean_recall": None,
|
||
"mean_f1": None,
|
||
"source": "fallback_no_detections",
|
||
"curve_point_count": 0,
|
||
},
|
||
"curve_paths": {},
|
||
"per_class": {
|
||
int(cls_id): {
|
||
"map50": None,
|
||
"map50_95": None,
|
||
"precision_ap": None,
|
||
"recall_ap": None,
|
||
"f1_at_recommended_confidence": None,
|
||
"optimal_confidence": fallback_threshold,
|
||
"optimal_precision": None,
|
||
"optimal_recall": None,
|
||
"optimal_f1": None,
|
||
}
|
||
for cls_id in unique_classes
|
||
},
|
||
}
|
||
|
||
tp = np.concatenate(store["tp"], axis=0)
|
||
conf = np.concatenate(store["conf"], axis=0)
|
||
pred_cls = np.concatenate(store["pred_cls"], axis=0)
|
||
resolved_save_dir = Path() if save_dir is None else Path(save_dir)
|
||
(
|
||
_tp,
|
||
_fp,
|
||
p,
|
||
r,
|
||
f1,
|
||
ap,
|
||
unique_classes,
|
||
p_curve,
|
||
r_curve,
|
||
f1_curve,
|
||
x,
|
||
_prec_values,
|
||
) = ap_per_class(
|
||
tp,
|
||
conf,
|
||
pred_cls,
|
||
target_cls,
|
||
plot=plot,
|
||
save_dir=resolved_save_dir,
|
||
names=names or {},
|
||
prefix=prefix,
|
||
)
|
||
best_index = int(smooth(f1_curve.mean(0), 0.1).argmax()) if f1_curve.size else 0
|
||
recommended_confidence = float(x[best_index]) if x.size else fallback_threshold
|
||
mean_precision = float(p.mean()) if p.size else None
|
||
mean_recall = float(r.mean()) if r.size else None
|
||
mean_f1 = float(f1.mean()) if f1.size else None
|
||
curve_paths = {}
|
||
if save_dir is not None:
|
||
candidate_paths = {
|
||
"pr_curve": resolved_save_dir / f"{prefix}PR_curve.png",
|
||
"f1_curve": resolved_save_dir / f"{prefix}F1_curve.png",
|
||
"precision_curve": resolved_save_dir / f"{prefix}P_curve.png",
|
||
"recall_curve": resolved_save_dir / f"{prefix}R_curve.png",
|
||
}
|
||
curve_paths = {key: str(path) for key, path in candidate_paths.items() if path.is_file()}
|
||
per_class = {}
|
||
for class_index, cls_id in enumerate(unique_classes.tolist()):
|
||
class_best_index = int(np.argmax(f1_curve[class_index])) if f1_curve.shape[1] else 0
|
||
per_class[int(cls_id)] = {
|
||
"map50": float(ap[class_index, 0]),
|
||
"map50_95": float(ap[class_index].mean()),
|
||
"precision_ap": float(p[class_index]),
|
||
"recall_ap": float(r[class_index]),
|
||
"f1_at_recommended_confidence": float(f1[class_index]),
|
||
"optimal_confidence": float(x[class_best_index]) if x.size else fallback_threshold,
|
||
"optimal_precision": float(p_curve[class_index, class_best_index]) if p_curve.shape[1] else None,
|
||
"optimal_recall": float(r_curve[class_index, class_best_index]) if r_curve.shape[1] else None,
|
||
"optimal_f1": float(f1_curve[class_index, class_best_index]) if f1_curve.shape[1] else None,
|
||
}
|
||
return {
|
||
"map50": float(ap[:, 0].mean()) if ap.size else None,
|
||
"map50_95": float(ap.mean()) if ap.size else None,
|
||
"threshold_advice": {
|
||
"recommended_confidence": recommended_confidence,
|
||
"mean_precision": mean_precision,
|
||
"mean_recall": mean_recall,
|
||
"mean_f1": mean_f1,
|
||
"source": "max_mean_f1",
|
||
"curve_point_count": int(x.size),
|
||
},
|
||
"curve_paths": curve_paths,
|
||
"per_class": per_class,
|
||
}
|
||
|
||
|
||
def summarize_2d_classification_from_confusion(confusion_matrix: ConfusionMatrix) -> dict[str, Any]:
|
||
nc = int(confusion_matrix.nc)
|
||
matrix = np.asarray(confusion_matrix.matrix, dtype=np.float64)
|
||
if matrix.shape[0] < nc or matrix.shape[1] < nc:
|
||
return {"overall_accuracy": None, "overall_pairs": 0, "per_class": {}, "matrix": matrix.tolist()}
|
||
|
||
matched_matrix = matrix[:nc, :nc]
|
||
overall_pairs = int(np.sum(matched_matrix))
|
||
overall_correct = float(np.trace(matched_matrix))
|
||
per_class: dict[int, dict[str, Any]] = {}
|
||
for cls_id in range(nc):
|
||
pairs = int(np.sum(matched_matrix[:, cls_id]))
|
||
correct = float(matched_matrix[cls_id, cls_id])
|
||
per_class[int(cls_id)] = {
|
||
"cls_eval_pairs_2d": pairs,
|
||
"cls_correct_2d": int(correct),
|
||
"cls_acc_2d": (correct / float(pairs)) if pairs > 0 else None,
|
||
}
|
||
return {
|
||
"overall_accuracy": (overall_correct / float(overall_pairs)) if overall_pairs > 0 else None,
|
||
"overall_pairs": overall_pairs,
|
||
"per_class": per_class,
|
||
"matrix": matrix.tolist(),
|
||
}
|
||
|
||
|
||
def make_label_accuracy_store(label_order: Iterable[str]) -> dict[str, Any]:
|
||
return {
|
||
"label_order": [str(label) for label in label_order],
|
||
"total": 0,
|
||
"correct": 0,
|
||
"per_label": defaultdict(lambda: {"total": 0, "correct": 0}),
|
||
"matrix": defaultdict(lambda: defaultdict(int)),
|
||
}
|
||
|
||
|
||
def add_label_accuracy_sample(store: dict[str, Any], gt_label: Optional[str], pred_label: Optional[str]) -> None:
|
||
if gt_label is None or pred_label is None:
|
||
return
|
||
gt_label = str(gt_label)
|
||
pred_label = str(pred_label)
|
||
store["total"] += 1
|
||
store["per_label"][gt_label]["total"] += 1
|
||
store["matrix"][gt_label][pred_label] += 1
|
||
if gt_label == pred_label:
|
||
store["correct"] += 1
|
||
store["per_label"][gt_label]["correct"] += 1
|
||
|
||
|
||
def summarize_label_accuracy_store(store: dict[str, Any]) -> dict[str, Any]:
|
||
label_order = [str(label) for label in store.get("label_order", [])]
|
||
seen_labels = set(label_order)
|
||
seen_labels.update(str(label) for label in store.get("per_label", {}).keys())
|
||
for gt_label, pred_counts in store.get("matrix", {}).items():
|
||
seen_labels.add(str(gt_label))
|
||
seen_labels.update(str(label) for label in pred_counts.keys())
|
||
ordered_labels = [label for label in label_order if label in seen_labels] + sorted(seen_labels - set(label_order))
|
||
per_label = {}
|
||
accuracies = []
|
||
for label in ordered_labels:
|
||
counts = store.get("per_label", {}).get(label, {})
|
||
total = int(counts.get("total", 0) or 0)
|
||
correct = int(counts.get("correct", 0) or 0)
|
||
accuracy = (correct / float(total)) if total > 0 else None
|
||
per_label[label] = {"total": total, "correct": correct, "accuracy": accuracy}
|
||
if accuracy is not None:
|
||
accuracies.append(float(accuracy))
|
||
matrix = {
|
||
gt_label: {pred_label: int(store.get("matrix", {}).get(gt_label, {}).get(pred_label, 0) or 0) for pred_label in ordered_labels}
|
||
for gt_label in ordered_labels
|
||
}
|
||
total = int(store.get("total", 0) or 0)
|
||
correct = int(store.get("correct", 0) or 0)
|
||
return {
|
||
"label_order": ordered_labels,
|
||
"overall_accuracy": (correct / float(total)) if total > 0 else None,
|
||
"mean_accuracy": mean_or_none(accuracies),
|
||
"total": total,
|
||
"correct": correct,
|
||
"per_label": per_label,
|
||
"matrix": matrix,
|
||
}
|
||
|
||
|
||
def label_accuracy_rows(summary: dict[str, Any], label_key: str = "label") -> list[dict[str, Any]]:
|
||
per_label = summary.get("per_label") or {}
|
||
return [
|
||
{
|
||
label_key: label,
|
||
"total": int((per_label.get(label) or {}).get("total", 0) or 0),
|
||
"correct": int((per_label.get(label) or {}).get("correct", 0) or 0),
|
||
"accuracy": (per_label.get(label) or {}).get("accuracy"),
|
||
}
|
||
for label in summary.get("label_order", [])
|
||
]
|
||
|
||
|
||
def label_confusion_matrix_rows(summary: dict[str, Any], label_key: str = "gt_label") -> list[dict[str, Any]]:
|
||
labels = [str(label) for label in summary.get("label_order", [])]
|
||
matrix = summary.get("matrix") or {}
|
||
rows = []
|
||
for gt_label in labels:
|
||
row = {label_key: gt_label}
|
||
pred_counts = matrix.get(gt_label) or {}
|
||
for pred_label in labels:
|
||
row[pred_label] = int(pred_counts.get(pred_label, 0) or 0)
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def face_type_label(face_type: Any) -> Optional[str]:
|
||
try:
|
||
face_type_int = int(face_type)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
return FACE_NAMES.get(face_type_int)
|
||
|
||
|
||
def cut_state_label(cut_state: Any) -> Optional[str]:
|
||
try:
|
||
cut_state_int = int(cut_state)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if cut_state_int == 1:
|
||
return "cut_in"
|
||
if cut_state_int == 2:
|
||
return "cut_out"
|
||
return None
|
||
|
||
|
||
def gt_face_selection_label(target_42: np.ndarray, gt_decoded: Optional[dict[str, Any]]) -> Optional[str]:
|
||
cut_label = cut_state_label(get_gt_cut_state(target_42))
|
||
if cut_label is not None:
|
||
return cut_label
|
||
if gt_decoded is None:
|
||
return None
|
||
return face_type_label(gt_decoded.get("visible_face_type"))
|
||
|
||
|
||
def pred_face_selection_label(pred_41: np.ndarray, pred_decoded: Optional[dict[str, Any]]) -> Optional[str]:
|
||
cut_label = cut_state_label(get_pred_cut_state(pred_41))
|
||
if cut_label is not None:
|
||
return cut_label
|
||
if pred_decoded is None:
|
||
return None
|
||
return face_type_label(pred_decoded.get("visible_face_type"))
|
||
|
||
|
||
def is_occluded_class_name(cls_name: Any) -> bool:
|
||
return str(cls_name).endswith("_fake")
|
||
|
||
|
||
def fake_class_label(cls_name: Any) -> str:
|
||
name = str(cls_name)
|
||
if name in {"car_fake", "bicyclist_fake", "pedestrian_fake", "car", "bicycle", "pedestrian"}:
|
||
return name
|
||
return "non_fake"
|
||
|
||
|
||
def occlusion_binary_label(cls_name: Any) -> str:
|
||
return "occluded" if is_occluded_class_name(cls_name) else "visible"
|
||
|
||
|
||
def angle_abs_deg(pred_rad: float, gt_rad: float) -> Optional[float]:
|
||
if pred_rad is None or gt_rad is None:
|
||
return None
|
||
if not (math.isfinite(pred_rad) and math.isfinite(gt_rad)):
|
||
return None
|
||
diff = (float(pred_rad) - float(gt_rad) + math.pi) % (2.0 * math.pi) - math.pi
|
||
diff = abs(diff)
|
||
diff = min(diff, abs(math.pi - diff))
|
||
return math.degrees(diff)
|
||
|
||
|
||
def depth_bin(depth: Optional[float]) -> str:
|
||
if depth is None or not math.isfinite(depth):
|
||
return "unknown"
|
||
if depth < 20.0:
|
||
return "<20m"
|
||
if depth < 40.0:
|
||
return "20-40m"
|
||
if depth < 60.0:
|
||
return "40-60m"
|
||
return ">=60m"
|
||
|
||
|
||
def bbox_diag_bin(diag_px: Optional[float]) -> str:
|
||
if diag_px is None or not math.isfinite(diag_px):
|
||
return "unknown"
|
||
if diag_px < 32.0:
|
||
return "<32px"
|
||
if diag_px < 64.0:
|
||
return "32-64px"
|
||
if diag_px < 128.0:
|
||
return "64-128px"
|
||
return ">=128px"
|
||
|
||
|
||
def sanitize_name(value: str) -> str:
|
||
safe = []
|
||
for char in str(value):
|
||
if char.isalnum() or char in ("-", "_", "."):
|
||
safe.append(char)
|
||
else:
|
||
safe.append("_")
|
||
return "".join(safe).strip("_") or "item"
|
||
|
||
|
||
def to_float(value: Any) -> Optional[float]:
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, (np.floating, np.integer)):
|
||
value = value.item()
|
||
try:
|
||
value = float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if not math.isfinite(value):
|
||
return None
|
||
return value
|
||
|
||
|
||
def to_list(values: Any) -> list[float]:
|
||
arr = np.asarray(values, dtype=np.float32).reshape(-1)
|
||
return [float(v) for v in arr.tolist()]
|
||
|
||
|
||
def to_serializable_array(values: Any) -> Any:
|
||
if values is None:
|
||
return None
|
||
arr = np.asarray(values)
|
||
if arr.ndim == 0:
|
||
return arr.item()
|
||
return arr.tolist()
|
||
|
||
|
||
def get_cls_name(names: Any, cls_id: int) -> str:
|
||
if isinstance(names, dict):
|
||
return str(names.get(cls_id, cls_id))
|
||
if isinstance(names, (list, tuple)) and 0 <= cls_id < len(names):
|
||
return str(names[cls_id])
|
||
return str(cls_id)
|
||
|
||
|
||
def names_to_dict(names: Any) -> dict[int, str]:
|
||
if isinstance(names, dict):
|
||
return {int(key): str(value) for key, value in names.items()}
|
||
if isinstance(names, (list, tuple)):
|
||
return {int(index): str(value) for index, value in enumerate(names)}
|
||
return {}
|
||
|
||
|
||
def format_float(value: Optional[float], digits: int = 3) -> str:
|
||
if value is None or not math.isfinite(value):
|
||
return "n/a"
|
||
return f"{value:.{digits}f}"
|
||
|
||
|
||
def format_percent(value: Optional[float], digits: int = 1) -> str:
|
||
if value is None or not math.isfinite(value):
|
||
return "n/a"
|
||
return f"{value * 100:.{digits}f}%"
|
||
|
||
|
||
def depth_interval_start(depth_m: Optional[float], bin_width_m: float) -> Optional[float]:
|
||
if depth_m is None or not math.isfinite(depth_m) or bin_width_m <= 0:
|
||
return None
|
||
depth = max(float(depth_m), 0.0)
|
||
return math.floor(depth / bin_width_m) * bin_width_m
|
||
|
||
|
||
def lateral_interval_start(lateral_m: Optional[float], bin_width_m: float, lateral_limit_m: float) -> Optional[float]:
|
||
if lateral_m is None or not math.isfinite(lateral_m) or bin_width_m <= 0:
|
||
return None
|
||
lateral = float(lateral_m)
|
||
limit = float(lateral_limit_m)
|
||
if lateral < -limit or lateral >= limit:
|
||
return None
|
||
return math.floor((lateral + limit) / bin_width_m) * bin_width_m - limit
|
||
|
||
|
||
def metric_interval_start(value: Optional[float], bin_width: float) -> Optional[float]:
|
||
if value is None or not math.isfinite(value) or bin_width <= 0:
|
||
return None
|
||
metric = max(float(value), 0.0)
|
||
return math.floor(metric / bin_width) * bin_width
|
||
|
||
|
||
def format_interval_bound(value: float) -> str:
|
||
if float(value).is_integer():
|
||
return str(int(value))
|
||
half_step = round(float(value) * 2.0)
|
||
if math.isclose(float(value), half_step / 2.0, rel_tol=0.0, abs_tol=1e-9):
|
||
return f"{float(value):.1f}"
|
||
return f"{float(value):.2f}"
|
||
|
||
|
||
def interval_label(start_m: float, bin_width_m: float, unit: str = "m") -> str:
|
||
end_m = start_m + bin_width_m
|
||
return f"[{format_interval_bound(start_m)},{format_interval_bound(end_m)}){unit}"
|
||
|
||
|
||
def interval_slug(start: float, bin_width: float, unit: str) -> str:
|
||
start_text = format_interval_bound(start).replace("-", "neg").replace(".", "p")
|
||
end_text = format_interval_bound(start + bin_width).replace("-", "neg").replace(".", "p")
|
||
return sanitize_name(f"{start_text}_{end_text}{unit}")
|
||
|
||
|
||
def make_interval_store() -> defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]]:
|
||
return defaultdict(list)
|
||
|
||
|
||
def add_interval_store_sample(
|
||
store: defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]],
|
||
cls_id: int,
|
||
cls_name: str,
|
||
start_value: Optional[float],
|
||
value: Optional[float],
|
||
relative_reference_value: Optional[float] = None,
|
||
relative_reference_abs: bool = False,
|
||
) -> None:
|
||
if start_value is None or value is None or not math.isfinite(value):
|
||
return
|
||
relative_reference = to_float(relative_reference_value)
|
||
if relative_reference is not None and math.isfinite(relative_reference):
|
||
relative_reference = abs(float(relative_reference)) if relative_reference_abs else float(relative_reference)
|
||
else:
|
||
relative_reference = None
|
||
store[(int(cls_id), str(cls_name), float(start_value))].append((float(value), relative_reference))
|
||
|
||
|
||
def add_interval_sample(
|
||
store: defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]],
|
||
cls_id: int,
|
||
cls_name: str,
|
||
depth_m: Optional[float],
|
||
value: Optional[float],
|
||
bin_width_m: float,
|
||
relative_reference_value: Optional[float] = None,
|
||
relative_reference_abs: bool = False,
|
||
) -> None:
|
||
start_m = depth_interval_start(depth_m, bin_width_m)
|
||
add_interval_store_sample(
|
||
store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
start_value=start_m,
|
||
value=value,
|
||
relative_reference_value=relative_reference_value,
|
||
relative_reference_abs=relative_reference_abs,
|
||
)
|
||
|
||
|
||
def add_lateral_interval_sample(
|
||
store: defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]],
|
||
cls_id: int,
|
||
cls_name: str,
|
||
lateral_m: Optional[float],
|
||
value: Optional[float],
|
||
bin_width_m: float,
|
||
lateral_limit_m: float,
|
||
relative_reference_value: Optional[float] = None,
|
||
relative_reference_abs: bool = False,
|
||
) -> None:
|
||
start_m = lateral_interval_start(lateral_m, bin_width_m, lateral_limit_m)
|
||
add_interval_store_sample(
|
||
store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
start_value=start_m,
|
||
value=value,
|
||
relative_reference_value=relative_reference_value,
|
||
relative_reference_abs=relative_reference_abs,
|
||
)
|
||
|
||
|
||
def add_heading_interval_sample(
|
||
store: defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]],
|
||
cls_id: int,
|
||
cls_name: str,
|
||
heading_deg: Optional[float],
|
||
value: Optional[float],
|
||
bin_width_deg: float,
|
||
relative_reference_value: Optional[float] = None,
|
||
relative_reference_abs: bool = False,
|
||
) -> None:
|
||
start_deg = heading_interval_start(heading_deg, bin_width_deg)
|
||
add_interval_store_sample(
|
||
store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
start_value=start_deg,
|
||
value=value,
|
||
relative_reference_value=relative_reference_value,
|
||
relative_reference_abs=relative_reference_abs,
|
||
)
|
||
|
||
|
||
def build_interval_rows(
|
||
store: defaultdict[tuple[int, str, float], list[tuple[float, Optional[float]]]],
|
||
bin_width_m: float,
|
||
value_name: str,
|
||
threshold: Optional[float] = None,
|
||
relative_value_name: Optional[str] = None,
|
||
relative_reference_abs: bool = False,
|
||
interval_prefix: str = "depth_bin",
|
||
interval_unit: str = "m",
|
||
relative_reference_label: str = "gt_bin",
|
||
relative_reference_unit: str = "m",
|
||
relative_reference_field: str = "relative_gt_bin_m",
|
||
) -> list[dict[str, Any]]:
|
||
rows = []
|
||
for (cls_id, cls_name, start_m), values in sorted(store.items(), key=lambda item: (item[0][0], item[0][2])):
|
||
sample_values: list[float] = []
|
||
relative_values: list[float] = []
|
||
relative_references: list[float] = []
|
||
bin_center_m = float(start_m + bin_width_m / 2.0)
|
||
for value_item in values:
|
||
if isinstance(value_item, tuple):
|
||
sample_value = to_float(value_item[0])
|
||
explicit_relative_reference = to_float(value_item[1])
|
||
else:
|
||
sample_value = to_float(value_item)
|
||
explicit_relative_reference = None
|
||
if sample_value is None or not math.isfinite(sample_value):
|
||
continue
|
||
sample_values.append(float(sample_value))
|
||
if relative_value_name:
|
||
effective_relative_reference = explicit_relative_reference
|
||
if effective_relative_reference is None:
|
||
effective_relative_reference = abs(bin_center_m) if relative_reference_abs else bin_center_m
|
||
if effective_relative_reference is not None and math.isfinite(effective_relative_reference):
|
||
effective_relative_reference = float(effective_relative_reference)
|
||
relative_references.append(effective_relative_reference)
|
||
if not math.isclose(effective_relative_reference, 0.0, rel_tol=0.0, abs_tol=1e-9):
|
||
relative_values.append(float(sample_value) * 100.0 / effective_relative_reference)
|
||
if not sample_values:
|
||
continue
|
||
arr = np.asarray(sample_values, dtype=np.float64)
|
||
bin_center_m = float(start_m + bin_width_m / 2.0)
|
||
interval_label_text = interval_label(float(start_m), float(bin_width_m), unit=interval_unit)
|
||
row = {
|
||
"cls_id": int(cls_id),
|
||
"cls_name": str(cls_name),
|
||
"interval_start": float(start_m),
|
||
"interval_end": float(start_m + bin_width_m),
|
||
"interval_center": bin_center_m,
|
||
"interval_label": interval_label_text,
|
||
f"{interval_prefix}_start_{interval_unit}": float(start_m),
|
||
f"{interval_prefix}_end_{interval_unit}": float(start_m + bin_width_m),
|
||
f"{interval_prefix}_center_{interval_unit}": bin_center_m,
|
||
f"{interval_prefix}_label": interval_label_text,
|
||
"count": int(arr.size),
|
||
f"mean_{value_name}": float(np.mean(arr)),
|
||
f"median_{value_name}": float(np.median(arr)),
|
||
f"p90_{value_name}": float(np.percentile(arr, 90)),
|
||
f"max_{value_name}": float(np.max(arr)),
|
||
}
|
||
if relative_value_name:
|
||
relative_arr = np.asarray(relative_values, dtype=np.float64)
|
||
relative_reference_value = mean_or_none(relative_references)
|
||
row["relative_reference_value"] = relative_reference_value
|
||
row["relative_reference_label"] = str(relative_reference_label)
|
||
row["relative_reference_unit"] = str(relative_reference_unit)
|
||
row[relative_reference_field] = relative_reference_value
|
||
if relative_arr.size > 0:
|
||
row[f"mean_{relative_value_name}"] = float(np.mean(relative_arr))
|
||
row[f"median_{relative_value_name}"] = float(np.median(relative_arr))
|
||
row[f"p90_{relative_value_name}"] = float(np.percentile(relative_arr, 90))
|
||
row[f"max_{relative_value_name}"] = float(np.max(relative_arr))
|
||
else:
|
||
row[f"mean_{relative_value_name}"] = None
|
||
row[f"median_{relative_value_name}"] = None
|
||
row[f"p90_{relative_value_name}"] = None
|
||
row[f"max_{relative_value_name}"] = None
|
||
if threshold is not None:
|
||
row[f"rate_gt_{str(threshold).replace('.', 'p')}"] = float(np.mean(arr > threshold))
|
||
row[f"count_gt_{str(threshold).replace('.', 'p')}"] = int(np.sum(arr > threshold))
|
||
rows.append(row)
|
||
return rows
|
||
|
||
|
||
def build_metric_bucket_interval_rows(
|
||
store: defaultdict[float, dict[str, Any]],
|
||
bin_width_m: float,
|
||
prefix: str = "lateral_bin",
|
||
) -> list[dict[str, Any]]:
|
||
rows = []
|
||
for start_m, bucket in sorted(store.items(), key=lambda item: float(item[0])):
|
||
summary = summarize_metric_bucket(bucket)
|
||
rows.append(
|
||
{
|
||
f"{prefix}_start_m": float(start_m),
|
||
f"{prefix}_end_m": float(start_m + bin_width_m),
|
||
f"{prefix}_label": interval_label(float(start_m), float(bin_width_m)),
|
||
**summary,
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def build_grouped_metric_bucket_interval_rows(
|
||
stores_by_group: dict[str, defaultdict[float, dict[str, Any]]],
|
||
bin_width_m: float,
|
||
prefix: str = "lateral_bin",
|
||
group_order: Optional[Iterable[str]] = None,
|
||
) -> dict[str, list[dict[str, Any]]]:
|
||
ordered_groups = [str(group) for group in (group_order or stores_by_group.keys())]
|
||
rows_by_group: dict[str, list[dict[str, Any]]] = {}
|
||
for group_name in ordered_groups:
|
||
rows_by_group[group_name] = build_metric_bucket_interval_rows(
|
||
stores_by_group.get(group_name, defaultdict(make_metric_bucket)),
|
||
bin_width_m=bin_width_m,
|
||
prefix=prefix,
|
||
)
|
||
return rows_by_group
|
||
|
||
|
||
def flatten_grouped_metric_bucket_interval_rows(
|
||
rows_by_group: dict[str, list[dict[str, Any]]],
|
||
group_key: str = "face_visibility_bucket",
|
||
) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
for group_name, group_rows in rows_by_group.items():
|
||
for row in group_rows:
|
||
rows.append({group_key: str(group_name), **row})
|
||
return rows
|
||
|
||
|
||
def write_rows_csv(path: Path, rows: list[dict[str, Any]], fallback_fields: Optional[list[str]] = None) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
if not rows:
|
||
with path.open("w", encoding="utf-8", newline="") as file:
|
||
writer = csv.writer(file)
|
||
writer.writerow(fallback_fields or [])
|
||
return
|
||
helper_fields = {
|
||
"interval_start",
|
||
"interval_end",
|
||
"interval_center",
|
||
"interval_label",
|
||
"relative_reference_value",
|
||
"relative_reference_label",
|
||
"relative_reference_unit",
|
||
}
|
||
fieldnames = [field for field in rows[0].keys() if field not in helper_fields]
|
||
with path.open("w", encoding="utf-8", newline="") as file:
|
||
writer = csv.DictWriter(file, fieldnames=fieldnames, extrasaction="ignore")
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def group_rows_by_class(rows: list[dict[str, Any]]) -> dict[tuple[int, str], list[dict[str, Any]]]:
|
||
grouped: dict[tuple[int, str], list[dict[str, Any]]] = defaultdict(list)
|
||
for row in rows:
|
||
grouped[(int(row["cls_id"]), str(row["cls_name"]))].append(row)
|
||
return grouped
|
||
|
||
|
||
def top_interval_rows(rows: list[dict[str, Any]], metric_key: str, topn: int = 3, min_count: int = 30) -> list[dict[str, Any]]:
|
||
filtered = [row for row in rows if row.get(metric_key) is not None and int(row.get("count", 0)) >= min_count]
|
||
if not filtered:
|
||
filtered = [row for row in rows if row.get(metric_key) is not None]
|
||
return sorted(filtered, key=lambda row: (float(row.get(metric_key) or 0.0), int(row.get("count", 0))), reverse=True)[:topn]
|
||
|
||
|
||
def make_metric_bucket() -> dict[str, Any]:
|
||
return {
|
||
"gt_total": 0,
|
||
"pred_total": 0,
|
||
"matched_2d": 0,
|
||
"matched_3d": 0,
|
||
"matched_pos": 0,
|
||
"yaw_compare_eligible_count": 0,
|
||
"yaw_compare_pair_count": 0,
|
||
"confidence": [],
|
||
"match_iou": [],
|
||
"yaw_abs_deg": [],
|
||
"x_abs_m": [],
|
||
"y_abs_m": [],
|
||
"z_abs_m": [],
|
||
"center_error_m": [],
|
||
"yaw_compare_yaw_abs_deg": [],
|
||
"direct_visible_yaw_abs_deg": [],
|
||
"edge_visible_yaw_abs_deg": [],
|
||
"direct_minus_edge_visible_yaw_abs_deg": [],
|
||
"length_compare_pair_count": 0,
|
||
"direct_length_abs_err_m": [],
|
||
"edge_length_abs_err_m": [],
|
||
"direct_minus_edge_length_abs_err_m": [],
|
||
"length_compare_edge_better_count": 0,
|
||
"length_compare_direct_better_count": 0,
|
||
"length_compare_tie_count": 0,
|
||
"yaw_compare_edge_better_count": 0,
|
||
"yaw_compare_direct_better_count": 0,
|
||
"yaw_compare_tie_count": 0,
|
||
"yaw_bad_count": 0,
|
||
"x_bad_count": 0,
|
||
"y_bad_count": 0,
|
||
"z_bad_count": 0,
|
||
"yaw_abs_sum": 0.0,
|
||
}
|
||
|
||
|
||
def add_prediction_count(bucket: dict[str, Any], count_value: int = 1) -> None:
|
||
bucket["pred_total"] += int(count_value)
|
||
|
||
|
||
def add_gt_count(bucket: dict[str, Any], count_value: int = 1) -> None:
|
||
bucket["gt_total"] += int(count_value)
|
||
|
||
|
||
def add_2d_match(bucket: dict[str, Any]) -> None:
|
||
bucket["matched_2d"] += 1
|
||
|
||
|
||
def add_3d_record(
|
||
bucket: dict[str, Any],
|
||
record: dict[str, Any],
|
||
yaw_bad_threshold_deg: float,
|
||
horizontal_bad_threshold_m: float,
|
||
vertical_bad_threshold_m: float,
|
||
edge_visible_key: str = "edge_visible_yaw_abs_deg",
|
||
direct_minus_edge_key: str = "direct_minus_edge_visible_yaw_abs_deg",
|
||
) -> None:
|
||
bucket["matched_3d"] += 1
|
||
bucket["confidence"].append(record["confidence"])
|
||
bucket["match_iou"].append(record["match_iou"])
|
||
|
||
yaw_abs = record["yaw_abs_deg"]
|
||
if yaw_abs is not None:
|
||
bucket["yaw_abs_deg"].append(yaw_abs)
|
||
bucket["yaw_abs_sum"] += yaw_abs
|
||
if yaw_abs >= yaw_bad_threshold_deg:
|
||
bucket["yaw_bad_count"] += 1
|
||
|
||
direct_visible = record["direct_visible_yaw_abs_deg"]
|
||
edge_visible = record.get(edge_visible_key)
|
||
if record["yaw_compare_eligible"]:
|
||
bucket["yaw_compare_eligible_count"] += 1
|
||
if direct_visible is not None and edge_visible is not None:
|
||
bucket["yaw_compare_pair_count"] += 1
|
||
if yaw_abs is not None:
|
||
bucket["yaw_compare_yaw_abs_deg"].append(yaw_abs)
|
||
bucket["direct_visible_yaw_abs_deg"].append(direct_visible)
|
||
bucket["edge_visible_yaw_abs_deg"].append(edge_visible)
|
||
direct_minus_edge = record.get(direct_minus_edge_key)
|
||
if direct_minus_edge is None:
|
||
direct_minus_edge = float(direct_visible) - float(edge_visible)
|
||
bucket["direct_minus_edge_visible_yaw_abs_deg"].append(direct_minus_edge)
|
||
eps = 1e-6
|
||
if edge_visible + eps < direct_visible:
|
||
bucket["yaw_compare_edge_better_count"] += 1
|
||
elif direct_visible + eps < edge_visible:
|
||
bucket["yaw_compare_direct_better_count"] += 1
|
||
else:
|
||
bucket["yaw_compare_tie_count"] += 1
|
||
|
||
direct_length_err = record.get("direct_length_abs_err_m")
|
||
edge_length_err = record.get("edge_length_abs_err_m")
|
||
if direct_length_err is not None and edge_length_err is not None:
|
||
bucket["length_compare_pair_count"] += 1
|
||
bucket["direct_length_abs_err_m"].append(float(direct_length_err))
|
||
bucket["edge_length_abs_err_m"].append(float(edge_length_err))
|
||
direct_minus_edge_length = record.get("direct_minus_edge_length_abs_err_m")
|
||
if direct_minus_edge_length is None:
|
||
direct_minus_edge_length = float(direct_length_err) - float(edge_length_err)
|
||
bucket["direct_minus_edge_length_abs_err_m"].append(float(direct_minus_edge_length))
|
||
eps = 1e-6
|
||
if edge_length_err + eps < direct_length_err:
|
||
bucket["length_compare_edge_better_count"] += 1
|
||
elif direct_length_err + eps < edge_length_err:
|
||
bucket["length_compare_direct_better_count"] += 1
|
||
else:
|
||
bucket["length_compare_tie_count"] += 1
|
||
|
||
if record["position_eligible"]:
|
||
bucket["matched_pos"] += 1
|
||
x_abs = record["x_abs_m"]
|
||
y_abs = record["y_abs_m"]
|
||
z_abs = record["z_abs_m"]
|
||
center = record["center_error_m"]
|
||
if x_abs is not None:
|
||
bucket["x_abs_m"].append(x_abs)
|
||
if x_abs > horizontal_bad_threshold_m:
|
||
bucket["x_bad_count"] += 1
|
||
if y_abs is not None:
|
||
bucket["y_abs_m"].append(y_abs)
|
||
if z_abs is not None:
|
||
bucket["z_abs_m"].append(z_abs)
|
||
if z_abs > vertical_bad_threshold_m:
|
||
bucket["z_bad_count"] += 1
|
||
if center is not None:
|
||
bucket["center_error_m"].append(center)
|
||
|
||
|
||
def mean_or_none(values: list[float]) -> Optional[float]:
|
||
if not values:
|
||
return None
|
||
return float(np.mean(np.asarray(values, dtype=np.float64)))
|
||
|
||
|
||
def percentile_or_none(values: list[float], q: float) -> Optional[float]:
|
||
if not values:
|
||
return None
|
||
return float(np.percentile(np.asarray(values, dtype=np.float64), q))
|
||
|
||
|
||
def rate_or_none(numer: int, denom: int) -> Optional[float]:
|
||
if denom <= 0:
|
||
return None
|
||
return float(numer) / float(denom)
|
||
|
||
|
||
def f1_or_none(precision: Optional[float], recall: Optional[float]) -> Optional[float]:
|
||
if precision is None or recall is None:
|
||
return None
|
||
denom = float(precision) + float(recall)
|
||
if denom <= 0.0:
|
||
return None
|
||
return (2.0 * float(precision) * float(recall)) / denom
|
||
|
||
|
||
def summarize_metric_bucket(bucket: dict[str, Any]) -> dict[str, Any]:
|
||
matched_3d = int(bucket["matched_3d"])
|
||
matched_pos = int(bucket["matched_pos"])
|
||
yaw_compare_count = int(bucket["yaw_compare_pair_count"])
|
||
length_compare_count = int(bucket["length_compare_pair_count"])
|
||
direct_regression_mae = mean_or_none(bucket["direct_visible_yaw_abs_deg"])
|
||
direct_regression_p90 = percentile_or_none(bucket["direct_visible_yaw_abs_deg"], 90.0)
|
||
edge_based_mae = mean_or_none(bucket["edge_visible_yaw_abs_deg"])
|
||
edge_based_p90 = percentile_or_none(bucket["edge_visible_yaw_abs_deg"], 90.0)
|
||
mean_direct_minus_edge = mean_or_none(bucket["direct_minus_edge_visible_yaw_abs_deg"])
|
||
median_direct_minus_edge = percentile_or_none(bucket["direct_minus_edge_visible_yaw_abs_deg"], 50.0)
|
||
direct_length_mae = mean_or_none(bucket["direct_length_abs_err_m"])
|
||
edge_length_mae = mean_or_none(bucket["edge_length_abs_err_m"])
|
||
mean_direct_minus_edge_length = mean_or_none(bucket["direct_minus_edge_length_abs_err_m"])
|
||
precision_2d = rate_or_none(bucket["matched_2d"], bucket["pred_total"])
|
||
recall_2d = rate_or_none(bucket["matched_2d"], bucket["gt_total"])
|
||
return {
|
||
"gt_total": int(bucket["gt_total"]),
|
||
"pred_total": int(bucket["pred_total"]),
|
||
"matched_2d": int(bucket["matched_2d"]),
|
||
"matched_3d": matched_3d,
|
||
"matched_pos": matched_pos,
|
||
"yaw_compare_eligible_count": int(bucket["yaw_compare_eligible_count"]),
|
||
"yaw_compare_count": yaw_compare_count,
|
||
"length_compare_count": length_compare_count,
|
||
"precision_2d": precision_2d,
|
||
"recall_2d": recall_2d,
|
||
"f1_2d": f1_or_none(precision_2d, recall_2d),
|
||
"mean_confidence": mean_or_none(bucket["confidence"]),
|
||
"mean_match_iou": mean_or_none(bucket["match_iou"]),
|
||
"yaw_mae_deg": mean_or_none(bucket["yaw_abs_deg"]),
|
||
"yaw_p90_deg": percentile_or_none(bucket["yaw_abs_deg"], 90.0),
|
||
"x_abs_mae_m": mean_or_none(bucket["x_abs_m"]),
|
||
"y_abs_mae_m": mean_or_none(bucket["y_abs_m"]),
|
||
"z_abs_mae_m": mean_or_none(bucket["z_abs_m"]),
|
||
"horizontal_abs_mae_m": mean_or_none(bucket["x_abs_m"]),
|
||
"vertical_abs_mae_m": mean_or_none(bucket["z_abs_m"]),
|
||
"center_error_mae_m": mean_or_none(bucket["center_error_m"]),
|
||
"yaw_compare_subset_yaw_mae_deg": mean_or_none(bucket["yaw_compare_yaw_abs_deg"]),
|
||
"yaw_compare_subset_yaw_p90_deg": percentile_or_none(bucket["yaw_compare_yaw_abs_deg"], 90.0),
|
||
"direct_visible_yaw_mae_deg": direct_regression_mae,
|
||
"direct_visible_yaw_p90_deg": direct_regression_p90,
|
||
"edge_visible_yaw_mae_deg": edge_based_mae,
|
||
"edge_visible_yaw_p90_deg": edge_based_p90,
|
||
"direct_regression_yaw_mae_deg": direct_regression_mae,
|
||
"direct_regression_yaw_p90_deg": direct_regression_p90,
|
||
"edge_based_yaw_mae_deg": edge_based_mae,
|
||
"edge_based_yaw_p90_deg": edge_based_p90,
|
||
"mean_direct_minus_edge_visible_yaw_deg": mean_direct_minus_edge,
|
||
"median_direct_minus_edge_visible_yaw_deg": median_direct_minus_edge,
|
||
"mean_direct_minus_edge_yaw_deg": mean_direct_minus_edge,
|
||
"median_direct_minus_edge_yaw_deg": median_direct_minus_edge,
|
||
"direct_regression_length_mae_m": direct_length_mae,
|
||
"side_edge_length_mae_m": edge_length_mae,
|
||
"mean_direct_minus_side_edge_length_m": mean_direct_minus_edge_length,
|
||
"length_compare_edge_better_count": int(bucket["length_compare_edge_better_count"]),
|
||
"length_compare_direct_better_count": int(bucket["length_compare_direct_better_count"]),
|
||
"length_compare_tie_count": int(bucket["length_compare_tie_count"]),
|
||
"yaw_compare_edge_better_count": int(bucket["yaw_compare_edge_better_count"]),
|
||
"yaw_compare_direct_better_count": int(bucket["yaw_compare_direct_better_count"]),
|
||
"yaw_compare_tie_count": int(bucket["yaw_compare_tie_count"]),
|
||
"yaw_bad_count": int(bucket["yaw_bad_count"]),
|
||
"x_bad_count": int(bucket["x_bad_count"]),
|
||
"y_bad_count": int(bucket["y_bad_count"]),
|
||
"z_bad_count": int(bucket["z_bad_count"]),
|
||
"horizontal_bad_count": int(bucket["x_bad_count"]),
|
||
"vertical_bad_count": int(bucket["z_bad_count"]),
|
||
"yaw_bad_rate": rate_or_none(bucket["yaw_bad_count"], matched_3d),
|
||
"x_bad_rate": rate_or_none(bucket["x_bad_count"], matched_pos),
|
||
"y_bad_rate": rate_or_none(bucket["y_bad_count"], matched_pos),
|
||
"z_bad_rate": rate_or_none(bucket["z_bad_count"], matched_pos),
|
||
"horizontal_bad_rate": rate_or_none(bucket["x_bad_count"], matched_pos),
|
||
"vertical_bad_rate": rate_or_none(bucket["z_bad_count"], matched_pos),
|
||
"length_compare_edge_better_rate": rate_or_none(bucket["length_compare_edge_better_count"], length_compare_count),
|
||
"length_compare_direct_better_rate": rate_or_none(bucket["length_compare_direct_better_count"], length_compare_count),
|
||
"length_compare_tie_rate": rate_or_none(bucket["length_compare_tie_count"], length_compare_count),
|
||
"yaw_compare_edge_better_rate": rate_or_none(bucket["yaw_compare_edge_better_count"], yaw_compare_count),
|
||
"yaw_compare_direct_better_rate": rate_or_none(bucket["yaw_compare_direct_better_count"], yaw_compare_count),
|
||
"yaw_compare_tie_rate": rate_or_none(bucket["yaw_compare_tie_count"], yaw_compare_count),
|
||
"yaw_abs_sum": float(bucket["yaw_abs_sum"]),
|
||
}
|
||
|
||
|
||
def make_breakdown_buckets() -> dict[str, dict[str, dict[str, Any]]]:
|
||
return {
|
||
"cut_status": defaultdict(make_metric_bucket),
|
||
"face_visibility": defaultdict(make_metric_bucket),
|
||
"distance_bin": defaultdict(make_metric_bucket),
|
||
"bbox_diag_bin": defaultdict(make_metric_bucket),
|
||
"class_group": defaultdict(make_metric_bucket),
|
||
}
|
||
|
||
|
||
def is_signed_lateral_yaw_compare_longitudinal_eligible(
|
||
record: dict[str, Any],
|
||
max_longitudinal_dist_m: float = DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M,
|
||
) -> bool:
|
||
longitudinal_m = to_float(record.get("gt_z_m"))
|
||
if longitudinal_m is None:
|
||
return False
|
||
return 0.0 <= float(longitudinal_m) < float(max_longitudinal_dist_m)
|
||
|
||
|
||
def make_reservoir_store() -> dict[str, Any]:
|
||
return {"seen": 0, "records": []}
|
||
|
||
|
||
def reservoir_add(
|
||
store: dict[str, Any],
|
||
record: dict[str, Any],
|
||
capacity: int,
|
||
rng: random.Random,
|
||
) -> None:
|
||
if capacity <= 0:
|
||
return
|
||
store["seen"] = int(store.get("seen", 0)) + 1
|
||
copied = dict(record)
|
||
records = store.setdefault("records", [])
|
||
if len(records) < capacity:
|
||
records.append(copied)
|
||
return
|
||
index = rng.randrange(int(store["seen"]))
|
||
if index < capacity:
|
||
records[index] = copied
|
||
|
||
|
||
def reservoir_records(store: dict[str, Any], rng: random.Random) -> list[dict[str, Any]]:
|
||
records = [dict(record) for record in store.get("records", [])]
|
||
rng.shuffle(records)
|
||
return records
|
||
|
||
|
||
def make_interval_visual_reservoirs() -> defaultdict[float, dict[str, Any]]:
|
||
return defaultdict(make_reservoir_store)
|
||
|
||
|
||
def add_interval_visual_record(
|
||
stores: defaultdict[float, dict[str, Any]],
|
||
value: Optional[float],
|
||
record: dict[str, Any],
|
||
bin_width: float,
|
||
per_bin_capacity: int,
|
||
rng: random.Random,
|
||
) -> None:
|
||
start = metric_interval_start(value, bin_width)
|
||
if start is None:
|
||
return
|
||
reservoir_add(stores[float(start)], record, per_bin_capacity, rng)
|
||
|
||
|
||
def make_writer(path: Path, fieldnames: Optional[list[str]] = None) -> tuple[Any, csv.DictWriter]:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
handle = path.open("w", encoding="utf-8", newline="")
|
||
writer = csv.DictWriter(handle, fieldnames=fieldnames or BADCASE_FIELDS)
|
||
writer.writeheader()
|
||
return handle, writer
|
||
|
||
|
||
def record_to_csv_row(record: dict[str, Any]) -> dict[str, Any]:
|
||
row = {key: record.get(key) for key in BADCASE_FIELDS}
|
||
row["gt_bbox_xyxy"] = json.dumps(record.get("gt_bbox_xyxy"))
|
||
row["pred_bbox_xyxy"] = json.dumps(record.get("pred_bbox_xyxy"))
|
||
return row
|
||
|
||
|
||
def record_2d_to_csv_row(record: dict[str, Any]) -> dict[str, Any]:
|
||
row = {key: record.get(key) for key in BADCASE_2D_FIELDS}
|
||
row["bbox_xyxy"] = json.dumps(record.get("bbox_xyxy"))
|
||
return row
|
||
|
||
|
||
def fake_class_record_to_csv_row(record: dict[str, Any]) -> dict[str, Any]:
|
||
row = {key: record.get(key) for key in FAKE_CLASS_BADCASE_FIELDS}
|
||
row["gt_bbox_xyxy"] = json.dumps(record.get("gt_bbox_xyxy"))
|
||
row["pred_bbox_xyxy"] = json.dumps(record.get("pred_bbox_xyxy"))
|
||
return row
|
||
|
||
|
||
def face_selection_record_to_csv_row(record: dict[str, Any]) -> dict[str, Any]:
|
||
row = {key: record.get(key) for key in FACE_SELECTION_BADCASE_FIELDS}
|
||
row["gt_bbox_xyxy"] = json.dumps(record.get("gt_bbox_xyxy"))
|
||
row["pred_bbox_xyxy"] = json.dumps(record.get("pred_bbox_xyxy"))
|
||
return row
|
||
|
||
|
||
def build_prediction_focus_mask(
|
||
detections: np.ndarray,
|
||
preds_3d: np.ndarray,
|
||
anchors: np.ndarray,
|
||
strides: np.ndarray,
|
||
calib: dict[str, Any],
|
||
max_abs_longitudinal_m: float,
|
||
) -> np.ndarray:
|
||
if len(detections) == 0:
|
||
return np.zeros((0,), dtype=bool)
|
||
focus_mask = np.zeros((len(detections),), dtype=bool)
|
||
for pred_index in range(len(detections)):
|
||
attrs = extract_3d_attrs_from_prediction(
|
||
preds_3d[pred_index],
|
||
anchors[:, pred_index],
|
||
float(strides[pred_index]),
|
||
calib,
|
||
)
|
||
center = None if attrs is None else attrs.get("center")
|
||
if center is None:
|
||
continue
|
||
center = np.asarray(center, dtype=np.float32).reshape(-1)
|
||
lateral_m = to_float(center[0]) if center.size > 0 else None
|
||
longitudinal_m = to_float(center[2]) if center.size > 2 else None
|
||
focus_mask[pred_index] = is_focused_confusion_spatial_object(
|
||
lateral_m=lateral_m,
|
||
longitudinal_m=longitudinal_m,
|
||
max_abs_longitudinal_m=max_abs_longitudinal_m,
|
||
)
|
||
return focus_mask
|
||
|
||
|
||
def iter_roi_analysis_samples(
|
||
bundle,
|
||
entries: list[tuple[str, str]],
|
||
image_root: Path,
|
||
class_map: dict[str, int],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
classes: Optional[set[int]],
|
||
min_wh_px: float,
|
||
inference_batch_size: int,
|
||
) -> Iterable[tuple[dict[str, Any], Any]]:
|
||
batch_size = max(1, int(inference_batch_size))
|
||
for indexed_entry_batch in iter_batches(enumerate(entries), batch_size):
|
||
batch_items = []
|
||
for sample_index, entry in indexed_entry_batch:
|
||
image_path = entry_to_image_file(entry, image_root)
|
||
label_path = entry_to_label_file(entry)
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
ori_h, ori_w = image_bgr.shape[:2]
|
||
raw_calib = read_raw_calib_from_label(image_path, label_path)
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
gt = prepare_gt_for_roi(
|
||
label_file=label_path,
|
||
ori_w=ori_w,
|
||
ori_h=ori_h,
|
||
prepared=prepared,
|
||
bundle=bundle,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
include_classes=classes,
|
||
min_wh_px=min_wh_px,
|
||
)
|
||
batch_items.append(
|
||
{
|
||
"sample_index": int(sample_index),
|
||
"image_path": image_path,
|
||
"label_path": label_path,
|
||
"prepared": prepared,
|
||
"gt": gt,
|
||
}
|
||
)
|
||
|
||
if not batch_items:
|
||
continue
|
||
|
||
raw_outputs_batch = run_model_for_prepared_roi_batch(
|
||
bundle=bundle,
|
||
prepared_batch=[item["prepared"] for item in batch_items],
|
||
)
|
||
for batch_item, raw_outputs in zip(batch_items, raw_outputs_batch):
|
||
yield batch_item, raw_outputs
|
||
|
||
|
||
def collect_2d_confidence_advice(
|
||
bundle,
|
||
entries: list[tuple[str, str]],
|
||
image_root: Path,
|
||
class_map: dict[str, int],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
classes: Optional[set[int]],
|
||
min_wh_px: float,
|
||
names_dict: dict[int, str],
|
||
roi_name: str,
|
||
roi_output: Path,
|
||
topk_badcases: int,
|
||
badcase_random_seed: int,
|
||
focused_confusion_max_abs_longitudinal_m: float,
|
||
threshold_search_conf: float,
|
||
configured_confidence_2d: float,
|
||
inference_batch_size: int,
|
||
log_every: int,
|
||
) -> tuple[dict[str, Any], dict[str, Any], float]:
|
||
ap_store_2d = make_2d_ap_store()
|
||
total_samples = len(entries)
|
||
start_time = time.time()
|
||
|
||
for batch_item, raw_outputs in iter_roi_analysis_samples(
|
||
bundle=bundle,
|
||
entries=entries,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
classes=classes,
|
||
min_wh_px=min_wh_px,
|
||
inference_batch_size=inference_batch_size,
|
||
):
|
||
sample_index = int(batch_item["sample_index"])
|
||
prepared = batch_item["prepared"]
|
||
gt = batch_item["gt"]
|
||
|
||
analysis_outputs = filter_prediction_outputs(
|
||
raw_outputs=raw_outputs,
|
||
conf_thres=float(threshold_search_conf),
|
||
max_det=bundle.spec.max_det,
|
||
classes=classes,
|
||
)
|
||
analysis_detections = analysis_outputs.detections
|
||
analysis_pred_boxes = (
|
||
np.asarray(analysis_detections[:, :4], dtype=np.float32) if len(analysis_detections) else np.zeros((0, 4), dtype=np.float32)
|
||
)
|
||
analysis_pred_cls = (
|
||
np.asarray(analysis_detections[:, 5], dtype=np.int32).reshape(-1) if len(analysis_detections) else np.zeros((0,), dtype=np.int32)
|
||
)
|
||
analysis_pred_conf = (
|
||
np.asarray(analysis_detections[:, 4], dtype=np.float32).reshape(-1) if len(analysis_detections) else np.zeros((0,), dtype=np.float32)
|
||
)
|
||
add_2d_ap_sample(
|
||
ap_store_2d,
|
||
gt_cls=gt["classes"],
|
||
gt_boxes=gt["boxes_xyxy"],
|
||
pred_cls=analysis_pred_cls,
|
||
pred_boxes=analysis_pred_boxes,
|
||
pred_conf=analysis_pred_conf,
|
||
iou_thresholds=np.linspace(0.5, 0.95, 10, dtype=np.float32),
|
||
)
|
||
|
||
if (sample_index + 1) % max(1, log_every) == 0 or (sample_index + 1) == total_samples:
|
||
elapsed = time.time() - start_time
|
||
per_sample = elapsed / max(sample_index + 1, 1)
|
||
remaining = total_samples - (sample_index + 1)
|
||
eta = remaining * per_sample
|
||
print(
|
||
f"[{roi_name}:advice] {sample_index + 1}/{total_samples} "
|
||
f"elapsed={elapsed / 60:.1f}m eta={eta / 60:.1f}m search_conf={threshold_search_conf:.3f}"
|
||
)
|
||
|
||
confidence_curve_output = roi_output / "2d_confidence_curves"
|
||
confidence_curve_output.mkdir(parents=True, exist_ok=True)
|
||
ap_summary_2d = summarize_2d_ap_store(
|
||
ap_store_2d,
|
||
plot=True,
|
||
save_dir=confidence_curve_output,
|
||
names=names_dict,
|
||
prefix="2d_",
|
||
)
|
||
threshold_advice_2d = ap_summary_2d.get("threshold_advice") or {}
|
||
report_confidence_2d = to_float(threshold_advice_2d.get("recommended_confidence"))
|
||
if report_confidence_2d is None:
|
||
report_confidence_2d = float(configured_confidence_2d)
|
||
print(
|
||
f"[{roi_name}:advice] configured_conf={configured_confidence_2d:.3f} "
|
||
f"recommended_conf={float(report_confidence_2d):.3f}"
|
||
)
|
||
return ap_summary_2d, threshold_advice_2d, float(report_confidence_2d)
|
||
|
||
|
||
def prepare_gt_for_roi(
|
||
label_file: Path,
|
||
ori_w: int,
|
||
ori_h: int,
|
||
prepared,
|
||
bundle,
|
||
class_map: dict[str, int],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
include_classes: Optional[set[int]],
|
||
min_wh_px: float,
|
||
) -> dict[str, Any]:
|
||
detailed_class_names = read_mapped_label_class_names(label_file, class_map)
|
||
lb_2d, lb_3d = parse_ground_3d_label_file(
|
||
str(label_file),
|
||
class_map,
|
||
difficulty_weights,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
min_wh=0.0,
|
||
)
|
||
detailed_class_names = filter_label_names_by_classes(detailed_class_names, lb_2d["cls"], include_classes)
|
||
lb_2d, lb_3d = filter_labels_by_classes(lb_2d, lb_3d, include_classes)
|
||
lb_2d, lb_3d = remap_labels_to_roi(lb_2d, lb_3d, ori_w, ori_h, prepared.crop_bounds)
|
||
detailed_class_names = filter_label_names_by_min_wh(
|
||
detailed_class_names,
|
||
lb_2d["bboxes"],
|
||
img_w=prepared.image.shape[1],
|
||
img_h=prepared.image.shape[0],
|
||
min_wh_px=float(min_wh_px),
|
||
)
|
||
lb_2d, lb_3d = filter_labels_by_min_wh(
|
||
lb_2d,
|
||
lb_3d,
|
||
img_w=prepared.image.shape[1],
|
||
img_h=prepared.image.shape[0],
|
||
min_wh_px=float(min_wh_px),
|
||
)
|
||
lb_3d = normalize_roi_depth(lb_3d, prepared.calib["fx"], bundle.spec.virtual_fx)
|
||
gt_boxes_xyxy = xywhn_to_xyxy(lb_2d["bboxes"], prepared.image.shape[1], prepared.image.shape[0])
|
||
gt_classes = lb_2d["cls"].reshape(-1).astype(np.int32)
|
||
if len(detailed_class_names) != len(gt_classes):
|
||
detailed_class_names = [get_cls_name(bundle.names, int(cls_id)) for cls_id in gt_classes.tolist()]
|
||
return {
|
||
"lb_2d": lb_2d,
|
||
"lb_3d": lb_3d,
|
||
"boxes_xyxy": gt_boxes_xyxy,
|
||
"classes": gt_classes,
|
||
"class_names": detailed_class_names,
|
||
}
|
||
|
||
|
||
def make_record(
|
||
sample_index: int,
|
||
roi_name: str,
|
||
image_path: Path,
|
||
label_path: Path,
|
||
cls_name: str,
|
||
gt_index: int,
|
||
pred_index: int,
|
||
gt_box: np.ndarray,
|
||
pred_box: np.ndarray,
|
||
match_iou: float,
|
||
prediction: dict[str, Any],
|
||
gt_attrs: dict[str, Any],
|
||
pred_attrs: dict[str, Any],
|
||
gt_position_attrs: Optional[dict[str, Any]],
|
||
pred_position_attrs: Optional[dict[str, Any]],
|
||
gt_decoded: dict[str, Any],
|
||
pred_decoded: dict[str, Any],
|
||
gt_visible_faces: list[tuple[int, np.ndarray]],
|
||
gt_visible_yaw: Optional[float],
|
||
pred_edge_visible_yaw: Optional[float],
|
||
pred_edge_bucket_visible_yaw: Optional[float] = None,
|
||
pred_edge_decoded: Optional[dict[str, Any]] = None,
|
||
pred_edge_box: Optional[dict[str, Any]] = None,
|
||
is_cut_object_flag: bool = False,
|
||
position_eligible: bool = False,
|
||
position_error_basis: Optional[str] = None,
|
||
yaw_compare_max_lateral_dist_m: float = 5.0,
|
||
pred_yaw_compare_face_types: Optional[Iterable[int]] = None,
|
||
pred_yaw_compare_valid: bool = False,
|
||
) -> dict[str, Any]:
|
||
gt_center = np.asarray(gt_attrs["center"], dtype=np.float32)
|
||
pred_center = np.asarray(pred_attrs["center"], dtype=np.float32)
|
||
gt_position_source = gt_position_attrs or gt_attrs
|
||
pred_position_source = pred_position_attrs or pred_attrs
|
||
gt_position_center = np.asarray(gt_position_source["center"], dtype=np.float32)
|
||
pred_position_center = np.asarray(pred_position_source["center"], dtype=np.float32)
|
||
position_error_basis = str(
|
||
position_error_basis
|
||
or (
|
||
"face_center"
|
||
if gt_position_source.get("face_center") is not None and pred_position_source.get("face_center") is not None
|
||
else "box_center"
|
||
)
|
||
)
|
||
gt_lateral_abs = to_float(abs(gt_center[0]))
|
||
x_abs = to_float(abs(pred_position_center[0] - gt_position_center[0]))
|
||
y_abs = to_float(abs(pred_position_center[1] - gt_position_center[1]))
|
||
z_abs = to_float(abs(pred_position_center[2] - gt_position_center[2]))
|
||
center_error = None
|
||
if np.all(np.isfinite(gt_position_center)) and np.all(np.isfinite(pred_position_center)):
|
||
center_error = float(np.linalg.norm(pred_position_center - gt_position_center))
|
||
gt_yaw = to_float(gt_attrs["yaw"])
|
||
pred_yaw = to_float(pred_attrs["yaw"])
|
||
gt_visible_yaw = to_float(gt_visible_yaw)
|
||
pred_edge_visible_yaw = to_float(pred_edge_visible_yaw)
|
||
pred_edge_bucket_visible_yaw = to_float(pred_edge_bucket_visible_yaw)
|
||
direct_visible_yaw_abs_deg = angle_abs_deg(pred_yaw, gt_yaw)
|
||
edge_visible_yaw_abs_deg = angle_abs_deg(pred_edge_visible_yaw, gt_yaw)
|
||
direct_minus_edge_visible_yaw_abs_deg = None
|
||
if direct_visible_yaw_abs_deg is not None and edge_visible_yaw_abs_deg is not None:
|
||
direct_minus_edge_visible_yaw_abs_deg = float(direct_visible_yaw_abs_deg) - float(edge_visible_yaw_abs_deg)
|
||
edge_bucket_visible_yaw_abs_deg = angle_abs_deg(pred_edge_bucket_visible_yaw, gt_yaw)
|
||
direct_minus_edge_bucket_visible_yaw_abs_deg = None
|
||
if direct_visible_yaw_abs_deg is not None and edge_bucket_visible_yaw_abs_deg is not None:
|
||
direct_minus_edge_bucket_visible_yaw_abs_deg = float(direct_visible_yaw_abs_deg) - float(edge_bucket_visible_yaw_abs_deg)
|
||
yaw_compare_eligible = bool(gt_lateral_abs is not None and gt_lateral_abs < float(yaw_compare_max_lateral_dist_m))
|
||
visible_face_names = [FACE_NAMES.get(face_type, str(face_type)) for face_type, _ in gt_visible_faces]
|
||
pred_compare_face_types = tuple(int(face_type) for face_type in (pred_yaw_compare_face_types or ()))
|
||
pred_compare_face_names = [FACE_NAMES.get(face_type, str(face_type)) for face_type in pred_compare_face_types]
|
||
pred_compare_has_side_face_visible = bool(any(face_type in (2, 3) for face_type in pred_compare_face_types))
|
||
pred_compare_bucket = classify_edge_yaw_prediction_bucket(pred_compare_face_types, pred_yaw_compare_valid)
|
||
bbox_w = float(max(gt_box[2] - gt_box[0], 0.0))
|
||
bbox_h = float(max(gt_box[3] - gt_box[1], 0.0))
|
||
bbox_diag = math.hypot(bbox_w, bbox_h)
|
||
gt_length_m = to_float(np.asarray(gt_attrs["dims"], dtype=np.float32)[0])
|
||
pred_direct_length_m = to_float(np.asarray(pred_attrs["dims"], dtype=np.float32)[0])
|
||
pred_edge_length_m = None
|
||
pred_edge_box_mode = None if pred_edge_box is None else pred_edge_box.get("mode")
|
||
if pred_edge_box is not None and pred_edge_box.get("dims") is not None and pred_edge_box_mode in ("two-face", "side"):
|
||
pred_edge_length_m = to_float(np.asarray(pred_edge_box["dims"], dtype=np.float32)[0])
|
||
direct_length_abs_err_m = None
|
||
edge_length_abs_err_m = None
|
||
direct_minus_edge_length_abs_err_m = None
|
||
if gt_length_m is not None and pred_direct_length_m is not None:
|
||
direct_length_abs_err_m = abs(float(pred_direct_length_m) - float(gt_length_m))
|
||
if gt_length_m is not None and pred_edge_length_m is not None:
|
||
edge_length_abs_err_m = abs(float(pred_edge_length_m) - float(gt_length_m))
|
||
if direct_length_abs_err_m is not None and edge_length_abs_err_m is not None:
|
||
direct_minus_edge_length_abs_err_m = float(direct_length_abs_err_m) - float(edge_length_abs_err_m)
|
||
|
||
gt_visible_face_type = None if gt_position_source.get("visible_face_type") is None else int(gt_position_source["visible_face_type"])
|
||
pred_visible_face_type = None if pred_position_source.get("visible_face_type") is None else int(pred_position_source["visible_face_type"])
|
||
gt_face_center = gt_position_source.get("face_center") if gt_position_source.get("face_center") is not None else gt_attrs.get("face_center")
|
||
pred_face_center = (
|
||
pred_position_source.get("face_center") if pred_position_source.get("face_center") is not None else pred_attrs.get("face_center")
|
||
)
|
||
gt_visible_face_types = () if gt_decoded is None else tuple(gt_decoded.get("visible_face_types", ()) or ())
|
||
pred_visible_face_types = () if pred_decoded is None else tuple(pred_decoded.get("visible_face_types", ()) or ())
|
||
pred_edge_visible_face_type = None if pred_edge_decoded is None else pred_edge_decoded.get("visible_face_type")
|
||
pred_edge_visible_face_types = () if pred_edge_decoded is None else tuple(pred_edge_decoded.get("visible_face_types", ()) or ())
|
||
pred_edge_corners = None if pred_edge_decoded is None else pred_edge_decoded.get("corners_3d")
|
||
pred_edge_points_2d = None if pred_edge_decoded is None else pred_edge_decoded.get("edge_points_2d")
|
||
pred_edge_dims = None if pred_edge_box is None else pred_edge_box.get("dims")
|
||
gt_decoded_visible_face_type = None if gt_decoded is None else gt_decoded.get("visible_face_type")
|
||
pred_decoded_visible_face_type = None if pred_decoded is None else pred_decoded.get("visible_face_type")
|
||
reuse_gt_decoded_face_overlay = gt_visible_face_type == gt_decoded_visible_face_type
|
||
reuse_pred_decoded_face_overlay = pred_visible_face_type == pred_decoded_visible_face_type
|
||
|
||
return {
|
||
"sample_index": int(sample_index),
|
||
"roi": roi_name,
|
||
"frame_name": image_path.name,
|
||
"image_path": str(image_path),
|
||
"label_path": str(label_path),
|
||
"cls_id": int(prediction["cls_id"]),
|
||
"cls_name": cls_name,
|
||
"gt_index": int(gt_index),
|
||
"pred_index": int(pred_index),
|
||
"confidence": float(prediction["confidence"]),
|
||
"match_iou": float(match_iou),
|
||
"bbox_diag_px": float(bbox_diag),
|
||
"bbox_diag_bin": bbox_diag_bin(bbox_diag),
|
||
"distance_bin": depth_bin(to_float(gt_center[2])),
|
||
"is_cut_object": bool(is_cut_object_flag),
|
||
"position_eligible": bool(position_eligible),
|
||
"position_error_basis": position_error_basis,
|
||
"yaw_compare_eligible": yaw_compare_eligible,
|
||
"visible_face_count": int(len(gt_visible_faces)),
|
||
"visible_faces": ",".join(visible_face_names) if visible_face_names else "none",
|
||
"has_side_face_visible": bool(any(face_type in (2, 3) for face_type, _ in gt_visible_faces)),
|
||
"yaw_compare_visible_face_count": int(len(pred_compare_face_types)),
|
||
"yaw_compare_visible_faces": ",".join(pred_compare_face_names) if pred_compare_face_names else "none",
|
||
"yaw_compare_has_side_face_visible": pred_compare_has_side_face_visible,
|
||
"yaw_compare_face_bucket": pred_compare_bucket,
|
||
"yaw_compare_pred_valid": bool(pred_yaw_compare_valid),
|
||
"yaw_abs_deg": angle_abs_deg(pred_yaw, gt_yaw),
|
||
"direct_visible_yaw_abs_deg": direct_visible_yaw_abs_deg,
|
||
"edge_visible_yaw_abs_deg": edge_visible_yaw_abs_deg,
|
||
"direct_minus_edge_visible_yaw_abs_deg": direct_minus_edge_visible_yaw_abs_deg,
|
||
"edge_bucket_visible_yaw_abs_deg": edge_bucket_visible_yaw_abs_deg,
|
||
"direct_minus_edge_bucket_visible_yaw_abs_deg": direct_minus_edge_bucket_visible_yaw_abs_deg,
|
||
"gt_length_m": gt_length_m,
|
||
"pred_direct_length_m": pred_direct_length_m,
|
||
"pred_edge_length_m": pred_edge_length_m,
|
||
"pred_edge_box_mode": pred_edge_box_mode,
|
||
"direct_length_abs_err_m": direct_length_abs_err_m,
|
||
"edge_length_abs_err_m": edge_length_abs_err_m,
|
||
"direct_minus_edge_length_abs_err_m": direct_minus_edge_length_abs_err_m,
|
||
"gt_lateral_abs_m": gt_lateral_abs,
|
||
"x_abs_m": x_abs,
|
||
"y_abs_m": y_abs,
|
||
"z_abs_m": z_abs,
|
||
"center_error_m": to_float(center_error),
|
||
"position_gt_x_m": to_float(gt_position_center[0]),
|
||
"position_pred_x_m": to_float(pred_position_center[0]),
|
||
"position_gt_y_m": to_float(gt_position_center[1]),
|
||
"position_pred_y_m": to_float(pred_position_center[1]),
|
||
"position_gt_z_m": to_float(gt_position_center[2]),
|
||
"position_pred_z_m": to_float(pred_position_center[2]),
|
||
"gt_x_m": to_float(gt_center[0]),
|
||
"pred_x_m": to_float(pred_center[0]),
|
||
"gt_y_m": to_float(gt_center[1]),
|
||
"pred_y_m": to_float(pred_center[1]),
|
||
"gt_z_m": to_float(gt_center[2]),
|
||
"pred_z_m": to_float(pred_center[2]),
|
||
"gt_depth_m": to_float(gt_attrs["depth"]),
|
||
"pred_depth_m": to_float(pred_attrs["depth"]),
|
||
"gt_yaw_deg": to_float(math.degrees(gt_yaw)) if gt_yaw is not None else None,
|
||
"pred_yaw_deg": to_float(math.degrees(pred_yaw)) if pred_yaw is not None else None,
|
||
"gt_visible_yaw_deg": to_float(math.degrees(gt_visible_yaw)) if gt_visible_yaw is not None else None,
|
||
"pred_edge_yaw_deg": to_float(math.degrees(pred_edge_visible_yaw)) if pred_edge_visible_yaw is not None else None,
|
||
"pred_edge_bucket_yaw_deg": to_float(math.degrees(pred_edge_bucket_visible_yaw)) if pred_edge_bucket_visible_yaw is not None else None,
|
||
"gt_bbox_xyxy": [float(v) for v in gt_box.tolist()],
|
||
"pred_bbox_xyxy": [float(v) for v in pred_box.tolist()],
|
||
"gt_center": to_list(gt_center),
|
||
"pred_center": to_list(pred_center),
|
||
"gt_position_center": to_list(gt_position_center),
|
||
"pred_position_center": to_list(pred_position_center),
|
||
"gt_dims": to_list(gt_attrs["dims"]),
|
||
"pred_dims": to_list(pred_attrs["dims"]),
|
||
"gt_face_center": None if gt_face_center is None else to_list(gt_face_center),
|
||
"pred_face_center": None if pred_face_center is None else to_list(pred_face_center),
|
||
"gt_visible_face_type": gt_visible_face_type,
|
||
"pred_visible_face_type": pred_visible_face_type,
|
||
"gt_visible_face_types": [int(v) for v in gt_visible_face_types],
|
||
"pred_visible_face_types": [int(v) for v in pred_visible_face_types],
|
||
"gt_face_center_2d": (
|
||
None if gt_decoded is None or not reuse_gt_decoded_face_overlay else to_serializable_array(gt_decoded.get("face_center_2d"))
|
||
),
|
||
"pred_face_center_2d": (
|
||
None if pred_decoded is None or not reuse_pred_decoded_face_overlay else to_serializable_array(pred_decoded.get("face_center_2d"))
|
||
),
|
||
"gt_face_color": None if gt_decoded is None or not reuse_gt_decoded_face_overlay else to_serializable_array(gt_decoded.get("face_color")),
|
||
"pred_face_color": (
|
||
None if pred_decoded is None or not reuse_pred_decoded_face_overlay else to_serializable_array(pred_decoded.get("face_color"))
|
||
),
|
||
"gt_edge_points_2d": None if gt_decoded is None else to_serializable_array(gt_decoded.get("edge_points_2d")),
|
||
"pred_edge_points_2d": None if pred_decoded is None else to_serializable_array(pred_decoded.get("edge_points_2d")),
|
||
"gt_corners": to_serializable_array(gt_attrs["corners_3d"]),
|
||
"pred_corners": to_serializable_array(pred_attrs["corners_3d"]),
|
||
"pred_edge_corners": None if pred_edge_corners is None else to_serializable_array(pred_edge_corners),
|
||
"pred_edge_dims": None if pred_edge_dims is None else to_list(pred_edge_dims),
|
||
"pred_edge_visible_face_type": None if pred_edge_visible_face_type is None else int(pred_edge_visible_face_type),
|
||
"pred_edge_visible_face_types": [int(v) for v in pred_edge_visible_face_types],
|
||
"pred_edge_edge_points_2d": None if pred_edge_points_2d is None else to_serializable_array(pred_edge_points_2d),
|
||
"gt_yaw_rad": gt_yaw,
|
||
"pred_yaw_rad": pred_yaw,
|
||
"gt_visible_yaw_rad": gt_visible_yaw,
|
||
"pred_edge_yaw_rad": pred_edge_visible_yaw,
|
||
"pred_edge_bucket_yaw_rad": pred_edge_bucket_visible_yaw,
|
||
}
|
||
|
||
|
||
def annotate_panel_title(image: np.ndarray, title: str) -> np.ndarray:
|
||
panel = image.copy()
|
||
cv2.putText(panel, title, (10, 24), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2, cv2.LINE_AA)
|
||
return panel
|
||
|
||
|
||
def draw_dashed_rectangle(
|
||
image: np.ndarray,
|
||
p1: tuple[int, int],
|
||
p2: tuple[int, int],
|
||
color: tuple[int, int, int],
|
||
thickness: int = 2,
|
||
dash: int = 8,
|
||
gap: int = 5,
|
||
) -> None:
|
||
x1, y1 = p1
|
||
x2, y2 = p2
|
||
for x in range(x1, x2, dash + gap):
|
||
cv2.line(image, (x, y1), (min(x + dash, x2), y1), color, thickness, cv2.LINE_AA)
|
||
cv2.line(image, (x, y2), (min(x + dash, x2), y2), color, thickness, cv2.LINE_AA)
|
||
for y in range(y1, y2, dash + gap):
|
||
cv2.line(image, (x1, y), (x1, min(y + dash, y2)), color, thickness, cv2.LINE_AA)
|
||
cv2.line(image, (x2, y), (x2, min(y + dash, y2)), color, thickness, cv2.LINE_AA)
|
||
|
||
|
||
def draw_box_with_label(image: np.ndarray, xyxy: list[float], color: tuple[int, int, int], label: str, dashed: bool = False) -> np.ndarray:
|
||
drawn = image.copy()
|
||
x1, y1, x2, y2 = [int(round(float(v))) for v in xyxy]
|
||
if dashed:
|
||
draw_dashed_rectangle(drawn, (x1, y1), (x2, y2), color, thickness=2)
|
||
else:
|
||
cv2.rectangle(drawn, (x1, y1), (x2, y2), color, 2, cv2.LINE_AA)
|
||
(tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
||
cv2.rectangle(drawn, (x1, max(0, y1 - th - 6)), (x1 + tw + 4, y1), color, -1)
|
||
cv2.putText(drawn, label, (x1 + 2, max(th + 1, y1 - 3)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1, cv2.LINE_AA)
|
||
return drawn
|
||
|
||
|
||
def parse_visible_face_names(visible_faces: Any) -> list[int]:
|
||
if visible_faces is None:
|
||
return []
|
||
if isinstance(visible_faces, str):
|
||
parts = [part.strip() for part in visible_faces.split(",") if part.strip() and part.strip() != "none"]
|
||
elif isinstance(visible_faces, (list, tuple)):
|
||
parts = [str(part).strip() for part in visible_faces if str(part).strip() and str(part).strip() != "none"]
|
||
else:
|
||
parts = [str(visible_faces).strip()]
|
||
inverse = {str(name): int(face_type) for face_type, name in FACE_NAMES.items()}
|
||
return [inverse[part] for part in parts if part in inverse]
|
||
|
||
|
||
def build_badcase_draw_kwargs(
|
||
corners_3d: Any,
|
||
calib: dict[str, Any],
|
||
visible_face_type: Optional[int] = None,
|
||
visible_face_types: Optional[list[int]] = None,
|
||
face_center_3d: Any = None,
|
||
) -> dict[str, Any]:
|
||
kwargs: dict[str, Any] = {}
|
||
corners_arr = np.asarray(corners_3d, dtype=np.float32)
|
||
if corners_arr.shape != (8, 3) or not np.isfinite(corners_arr).all():
|
||
return kwargs
|
||
|
||
edge_face_types = [int(face_type) for face_type in (visible_face_types or []) if face_type in range(4)]
|
||
if not edge_face_types and visible_face_type in range(4):
|
||
edge_face_types = [int(visible_face_type)]
|
||
if edge_face_types:
|
||
_, edge_points_2d = collect_face_bottom_edges(corners_arr, edge_face_types, calib, num_samples=5)
|
||
kwargs["edge_points_2d"] = edge_points_2d
|
||
|
||
face_type = int(visible_face_type) if visible_face_type in range(4) else None
|
||
if face_type is not None:
|
||
if face_center_3d is None:
|
||
face_center_3d = face_center_from_corners(corners_arr, face_type)
|
||
face_center_arr = None if face_center_3d is None else np.asarray(face_center_3d, dtype=np.float32)
|
||
if face_center_arr is not None and face_center_arr.shape == (3,) and np.isfinite(face_center_arr).all():
|
||
projected = project_3d_to_2d_with_distortion(face_center_arr[None, :], calib)
|
||
if projected is not None and len(projected) > 0 and np.isfinite(projected[0]).all():
|
||
kwargs["face_center_2d"] = tuple(float(v) for v in projected[0].tolist())
|
||
kwargs["face_color"] = FACE_COLORS[face_type]
|
||
return kwargs
|
||
|
||
|
||
def draw_3d_panel_object(
|
||
panel: np.ndarray,
|
||
calib: dict[str, Any],
|
||
corners_3d: Any,
|
||
dims: list[float],
|
||
yaw_rad: Optional[float],
|
||
visible_face_type: Optional[int] = None,
|
||
face_center: Any = None,
|
||
visible_face_types: Optional[list[int]] = None,
|
||
face_center_2d: Any = None,
|
||
face_color: Any = None,
|
||
edge_points_2d: Any = None,
|
||
rebuild: bool = False,
|
||
thickness: int = 2,
|
||
) -> np.ndarray:
|
||
dims_arr = np.asarray(dims, dtype=np.float32)
|
||
corners_arr = np.asarray(corners_3d, dtype=np.float32)
|
||
if corners_arr.shape != (8, 3) or not np.all(np.isfinite(corners_arr)):
|
||
return panel
|
||
|
||
draw_corners = corners_arr
|
||
if rebuild and yaw_rad is not None and np.all(np.isfinite(dims_arr)):
|
||
rebuilt_corners = rebuild_box_corners_for_visualization(
|
||
corners_arr,
|
||
dims_arr,
|
||
float(yaw_rad),
|
||
visible_face_type=visible_face_type,
|
||
face_center_3d=face_center,
|
||
)
|
||
if rebuilt_corners is not None:
|
||
draw_corners = np.asarray(rebuilt_corners, dtype=np.float32)
|
||
draw_kwargs = build_badcase_draw_kwargs(
|
||
draw_corners,
|
||
calib,
|
||
visible_face_type=visible_face_type,
|
||
visible_face_types=visible_face_types,
|
||
face_center_3d=face_center,
|
||
)
|
||
if face_center_2d is not None:
|
||
face_center_2d_arr = np.asarray(face_center_2d, dtype=np.float32).reshape(-1)
|
||
if face_center_2d_arr.size >= 2 and np.isfinite(face_center_2d_arr[:2]).all():
|
||
draw_kwargs["face_center_2d"] = tuple(float(v) for v in face_center_2d_arr[:2].tolist())
|
||
if face_color is not None:
|
||
face_color_arr = np.asarray(face_color, dtype=np.int32).reshape(-1)
|
||
if face_color_arr.size >= 3:
|
||
draw_kwargs["face_color"] = tuple(int(v) for v in face_color_arr[:3].tolist())
|
||
if edge_points_2d is not None:
|
||
draw_kwargs["edge_points_2d"] = np.asarray(edge_points_2d, dtype=np.float32)
|
||
draw_3d_box(panel, draw_corners, calib, thickness=thickness, **draw_kwargs)
|
||
return panel
|
||
|
||
|
||
def make_3d_panel(
|
||
image: np.ndarray,
|
||
calib: dict[str, Any],
|
||
corners_3d: Any,
|
||
dims: list[float],
|
||
yaw_rad: Optional[float],
|
||
title: str,
|
||
visible_face_type: Optional[int] = None,
|
||
face_center: Any = None,
|
||
visible_face_types: Optional[list[int]] = None,
|
||
face_center_2d: Any = None,
|
||
face_color: Any = None,
|
||
edge_points_2d: Any = None,
|
||
rebuild: bool = False,
|
||
) -> np.ndarray:
|
||
panel = image.copy()
|
||
draw_3d_panel_object(
|
||
panel,
|
||
calib,
|
||
corners_3d,
|
||
dims,
|
||
yaw_rad,
|
||
visible_face_type=visible_face_type,
|
||
face_center=face_center,
|
||
visible_face_types=visible_face_types,
|
||
face_center_2d=face_center_2d,
|
||
face_color=face_color,
|
||
edge_points_2d=edge_points_2d,
|
||
rebuild=rebuild,
|
||
thickness=2,
|
||
)
|
||
return annotate_panel_title(panel, title)
|
||
|
||
|
||
def make_text_panel(shape: tuple[int, int, int], title: str, record: dict[str, Any]) -> np.ndarray:
|
||
panel = np.full(shape, 28, dtype=np.uint8)
|
||
lines = [
|
||
title,
|
||
f"{record['cls_name']} conf={record['confidence']:.2f} iou={record['match_iou']:.2f}",
|
||
f"face_select gt={record.get('gt_face_selection_label', 'n/a')} pred={record.get('pred_face_selection_label', 'n/a')}"
|
||
if record.get("gt_face_selection_label") is not None or record.get("pred_face_selection_label") is not None
|
||
else "face_select n/a",
|
||
f"yaw gt={record['gt_yaw_deg']:.1f} direct={record['pred_yaw_deg']:.1f} err={record['yaw_abs_deg']:.1f} deg"
|
||
if record["yaw_abs_deg"] is not None and record["gt_yaw_deg"] is not None and record["pred_yaw_deg"] is not None
|
||
else "yaw unavailable",
|
||
(
|
||
f"yaw gt={record['gt_yaw_deg']:.1f} edge={record['pred_edge_yaw_deg']:.1f}"
|
||
)
|
||
if record["gt_yaw_deg"] is not None and record["pred_edge_yaw_deg"] is not None
|
||
else "edge yaw unavailable",
|
||
(
|
||
f"gt-yaw err direct={record['direct_visible_yaw_abs_deg']:.1f} edge={record['edge_visible_yaw_abs_deg']:.1f} "
|
||
f"gain={record['direct_minus_edge_visible_yaw_abs_deg']:.1f}"
|
||
)
|
||
if record["direct_visible_yaw_abs_deg"] is not None
|
||
and record["edge_visible_yaw_abs_deg"] is not None
|
||
and record["direct_minus_edge_visible_yaw_abs_deg"] is not None
|
||
else "paired yaw comparison unavailable",
|
||
(
|
||
f"|gt_x|={record['gt_lateral_abs_m']:.2f}m yaw_compare={record['yaw_compare_eligible']} "
|
||
f"cut={record['is_cut_object']} pos_eligible={record['position_eligible']} basis={record.get('position_error_basis', 'n/a')}"
|
||
)
|
||
if record["gt_lateral_abs_m"] is not None
|
||
else (
|
||
f"yaw_compare={record['yaw_compare_eligible']} cut={record['is_cut_object']} "
|
||
f"pos_eligible={record['position_eligible']} basis={record.get('position_error_basis', 'n/a')}"
|
||
),
|
||
f"x gt={record['position_gt_x_m']:.2f} pred={record['position_pred_x_m']:.2f} abs={record['x_abs_m']:.2f} m"
|
||
if record["x_abs_m"] is not None and record["position_gt_x_m"] is not None and record["position_pred_x_m"] is not None
|
||
else "x unavailable",
|
||
f"y gt={record['position_gt_y_m']:.2f} pred={record['position_pred_y_m']:.2f} abs={record['y_abs_m']:.2f} m"
|
||
if record["y_abs_m"] is not None and record["position_gt_y_m"] is not None and record["position_pred_y_m"] is not None
|
||
else "y unavailable",
|
||
f"z gt={record['position_gt_z_m']:.2f} pred={record['position_pred_z_m']:.2f} abs={record['z_abs_m']:.2f} m"
|
||
if record["z_abs_m"] is not None and record["position_gt_z_m"] is not None and record["position_pred_z_m"] is not None
|
||
else "z unavailable",
|
||
f"faces={record['visible_face_count']} [{record['visible_faces']}]",
|
||
f"dist={record['distance_bin']} box={record['bbox_diag_bin']} frame={record['frame_name']}",
|
||
]
|
||
y = 26
|
||
for line in lines:
|
||
cv2.putText(panel, line, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.54, (220, 220, 220), 1, cv2.LINE_AA)
|
||
y += 34
|
||
return panel
|
||
|
||
|
||
def metric_value_for_category(record: dict[str, Any], category: str) -> Optional[float]:
|
||
if category == "yaw":
|
||
return to_float(record.get("yaw_abs_deg"))
|
||
if category == "horizontal":
|
||
return to_float(record.get("x_abs_m"))
|
||
if category == "vertical":
|
||
return to_float(record.get("z_abs_m"))
|
||
return None
|
||
|
||
|
||
def metric_unit_for_category(category: str) -> str:
|
||
return "deg" if category == "yaw" else "m"
|
||
|
||
|
||
def make_interval_bin_text_panel(
|
||
shape: tuple[int, int, int],
|
||
title: str,
|
||
category: str,
|
||
threshold_start: float,
|
||
threshold_label: str,
|
||
records: list[dict[str, Any]],
|
||
) -> np.ndarray:
|
||
panel = np.full(shape, 28, dtype=np.uint8)
|
||
unit = metric_unit_for_category(category)
|
||
metric_values = [metric_value_for_category(record, category) for record in records]
|
||
metric_values = [float(value) for value in metric_values if value is not None]
|
||
cls_names = sorted({str(record.get("cls_name", "unknown")) for record in records})
|
||
lines = [
|
||
title,
|
||
f"objects with {category} error >= {format_float(threshold_start, 1 if unit == 'm' else 0)} {unit}: {len(records)}",
|
||
f"bin={threshold_label} max={format_float(max(metric_values) if metric_values else None, 2)}{unit} mean={format_float(mean_or_none(metric_values), 2)}{unit}",
|
||
f"classes={','.join(cls_names[:4])}" + (f" +{max(0, len(cls_names) - 4)} more" if len(cls_names) > 4 else ""),
|
||
f"frame={records[0]['frame_name']}" if records else "frame=n/a",
|
||
]
|
||
for index, record in enumerate(records[:8], start=1):
|
||
metric_value = metric_value_for_category(record, category)
|
||
lines.append(
|
||
f"#{index} {record['cls_name']} err={format_float(metric_value, 2)}{unit} conf={format_float(record.get('confidence'), 2)} iou={format_float(record.get('match_iou'), 2)}"
|
||
)
|
||
if len(records) > 8:
|
||
lines.append(f"... and {len(records) - 8} more objects")
|
||
y = 26
|
||
for line in lines[:11]:
|
||
cv2.putText(panel, line, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.54, (220, 220, 220), 1, cv2.LINE_AA)
|
||
y += 34
|
||
return panel
|
||
|
||
|
||
def make_2d_badcase_record(
|
||
sample_index: int,
|
||
roi_name: str,
|
||
image_path: Path,
|
||
label_path: Path,
|
||
kind: str,
|
||
cls_id: int,
|
||
cls_name: str,
|
||
bbox_xyxy: np.ndarray,
|
||
gt_index: Optional[int] = None,
|
||
pred_index: Optional[int] = None,
|
||
confidence: Optional[float] = None,
|
||
max_iou_any: Optional[float] = None,
|
||
max_iou_same_class: Optional[float] = None,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"sample_index": int(sample_index),
|
||
"roi": roi_name,
|
||
"frame_name": image_path.name,
|
||
"image_path": str(image_path),
|
||
"label_path": str(label_path),
|
||
"kind": str(kind),
|
||
"cls_id": int(cls_id),
|
||
"cls_name": str(cls_name),
|
||
"gt_index": None if gt_index is None else int(gt_index),
|
||
"pred_index": None if pred_index is None else int(pred_index),
|
||
"confidence": to_float(confidence),
|
||
"max_iou_any": to_float(max_iou_any),
|
||
"max_iou_same_class": to_float(max_iou_same_class),
|
||
"bbox_xyxy": [float(v) for v in np.asarray(bbox_xyxy, dtype=np.float32).tolist()],
|
||
}
|
||
|
||
|
||
def make_face_selection_badcase_record(
|
||
sample_index: int,
|
||
roi_name: str,
|
||
image_path: Path,
|
||
label_path: Path,
|
||
gt_index: int,
|
||
pred_index: int,
|
||
gt_cls_id: int,
|
||
gt_cls_name: str,
|
||
pred_cls_id: int,
|
||
pred_cls_name: str,
|
||
gt_face_selection_label: str,
|
||
pred_face_selection_label: str,
|
||
gt_bbox_xyxy: np.ndarray,
|
||
pred_bbox_xyxy: np.ndarray,
|
||
match_iou: float,
|
||
confidence: float,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"sample_index": int(sample_index),
|
||
"roi": roi_name,
|
||
"frame_name": image_path.name,
|
||
"image_path": str(image_path),
|
||
"label_path": str(label_path),
|
||
"gt_index": int(gt_index),
|
||
"pred_index": int(pred_index),
|
||
"gt_cls_id": int(gt_cls_id),
|
||
"gt_cls_name": str(gt_cls_name),
|
||
"pred_cls_id": int(pred_cls_id),
|
||
"pred_cls_name": str(pred_cls_name),
|
||
"gt_face_selection_label": str(gt_face_selection_label),
|
||
"pred_face_selection_label": str(pred_face_selection_label),
|
||
"match_iou": float(match_iou),
|
||
"confidence": to_float(confidence),
|
||
"gt_bbox_xyxy": [float(v) for v in np.asarray(gt_bbox_xyxy, dtype=np.float32).tolist()],
|
||
"pred_bbox_xyxy": [float(v) for v in np.asarray(pred_bbox_xyxy, dtype=np.float32).tolist()],
|
||
}
|
||
|
||
|
||
def make_fake_class_badcase_record(
|
||
sample_index: int,
|
||
roi_name: str,
|
||
image_path: Path,
|
||
label_path: Path,
|
||
gt_index: int,
|
||
pred_index: int,
|
||
gt_cls_id: int,
|
||
gt_cls_name: str,
|
||
pred_cls_id: int,
|
||
pred_cls_name: str,
|
||
gt_bbox_xyxy: np.ndarray,
|
||
pred_bbox_xyxy: np.ndarray,
|
||
match_iou: float,
|
||
confidence: float,
|
||
) -> dict[str, Any]:
|
||
gt_fake_label = fake_class_label(gt_cls_name)
|
||
pred_fake_label = fake_class_label(pred_cls_name)
|
||
gt_occ = occlusion_binary_label(gt_cls_name)
|
||
pred_occ = occlusion_binary_label(pred_cls_name)
|
||
if gt_fake_label == pred_fake_label:
|
||
kind = "fake_class_correct"
|
||
elif gt_fake_label == "non_fake" or pred_fake_label == "non_fake":
|
||
kind = "fake_vs_non_fake"
|
||
elif gt_occ == "occluded" and pred_occ == "occluded":
|
||
kind = "fake_internal_confusion"
|
||
else:
|
||
kind = "visible_internal_confusion"
|
||
return {
|
||
"sample_index": int(sample_index),
|
||
"roi": roi_name,
|
||
"frame_name": image_path.name,
|
||
"image_path": str(image_path),
|
||
"label_path": str(label_path),
|
||
"kind": kind,
|
||
"gt_index": int(gt_index),
|
||
"pred_index": int(pred_index),
|
||
"gt_cls_id": int(gt_cls_id),
|
||
"gt_cls_name": str(gt_cls_name),
|
||
"pred_cls_id": int(pred_cls_id),
|
||
"pred_cls_name": str(pred_cls_name),
|
||
"gt_fake_class_label": gt_fake_label,
|
||
"pred_fake_class_label": pred_fake_label,
|
||
"gt_occlusion_label": gt_occ,
|
||
"pred_occlusion_label": pred_occ,
|
||
"match_iou": float(match_iou),
|
||
"confidence": to_float(confidence),
|
||
"gt_bbox_xyxy": [float(v) for v in np.asarray(gt_bbox_xyxy, dtype=np.float32).tolist()],
|
||
"pred_bbox_xyxy": [float(v) for v in np.asarray(pred_bbox_xyxy, dtype=np.float32).tolist()],
|
||
}
|
||
|
||
|
||
def make_fake_class_text_panel(shape: tuple[int, int, int], title: str, record: dict[str, Any]) -> np.ndarray:
|
||
panel = np.full(shape, 28, dtype=np.uint8)
|
||
lines = [
|
||
title,
|
||
f"kind={record['kind']}",
|
||
f"GT {record['gt_cls_name']} fake={record.get('gt_fake_class_label')} occ={record['gt_occlusion_label']}",
|
||
f"Pred {record['pred_cls_name']} fake={record.get('pred_fake_class_label')} occ={record['pred_occlusion_label']} conf={format_float(record.get('confidence'), 2)}",
|
||
f"iou={format_float(record.get('match_iou'), 3)} gt_index={record.get('gt_index')} pred_index={record.get('pred_index')}",
|
||
f"frame={record['frame_name']}",
|
||
]
|
||
y = 28
|
||
for line in lines:
|
||
cv2.putText(panel, line, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.56, (220, 220, 220), 1, cv2.LINE_AA)
|
||
y += 34
|
||
return panel
|
||
|
||
|
||
def make_2d_badcase_text_panel(shape: tuple[int, int, int], title: str, record: dict[str, Any]) -> np.ndarray:
|
||
panel = np.full(shape, 28, dtype=np.uint8)
|
||
lines = [
|
||
title,
|
||
f"{record['cls_name']} kind={record['kind']}",
|
||
f"conf={format_float(record.get('confidence'), 2)}" if record.get("confidence") is not None else "conf=n/a",
|
||
f"max_iou_any={format_float(record.get('max_iou_any'), 3)}",
|
||
f"max_iou_same_class={format_float(record.get('max_iou_same_class'), 3)}",
|
||
f"gt_index={record.get('gt_index')} pred_index={record.get('pred_index')}",
|
||
f"frame={record['frame_name']}",
|
||
]
|
||
y = 28
|
||
for line in lines:
|
||
cv2.putText(panel, line, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.56, (220, 220, 220), 1, cv2.LINE_AA)
|
||
y += 34
|
||
return panel
|
||
|
||
|
||
def make_face_selection_text_panel(shape: tuple[int, int, int], title: str, record: dict[str, Any]) -> np.ndarray:
|
||
panel = np.full(shape, 28, dtype=np.uint8)
|
||
lines = [
|
||
title,
|
||
f"GT select={record.get('gt_face_selection_label')} cls={record.get('gt_cls_name')}",
|
||
f"Pred select={record.get('pred_face_selection_label')} cls={record.get('pred_cls_name')} conf={format_float(record.get('confidence'), 2)}",
|
||
f"iou={format_float(record.get('match_iou'), 3)} gt_index={record.get('gt_index')} pred_index={record.get('pred_index')}",
|
||
f"frame={record['frame_name']}",
|
||
]
|
||
y = 28
|
||
for line in lines:
|
||
cv2.putText(panel, line, (10, y), cv2.FONT_HERSHEY_SIMPLEX, 0.56, (220, 220, 220), 1, cv2.LINE_AA)
|
||
y += 34
|
||
return panel
|
||
|
||
|
||
def resize_panel(panel: np.ndarray, target_size: tuple[int, int]) -> np.ndarray:
|
||
target_w, target_h = target_size
|
||
if panel.shape[1] == target_w and panel.shape[0] == target_h:
|
||
return panel
|
||
return cv2.resize(panel, (target_w, target_h), interpolation=cv2.INTER_LINEAR)
|
||
|
||
|
||
def assemble_visual_grid(panels: list[np.ndarray], panel_size: tuple[int, int]) -> np.ndarray:
|
||
resized = [resize_panel(panel, panel_size) for panel in panels]
|
||
top = np.concatenate(resized[:3], axis=1)
|
||
bottom = np.concatenate(resized[3:], axis=1)
|
||
return np.concatenate([top, bottom], axis=0)
|
||
|
||
|
||
def save_badcase_visuals(
|
||
records: list[dict[str, Any]],
|
||
output_dir: Path,
|
||
category: str,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest = []
|
||
for rank, record in enumerate(records, start=1):
|
||
image_path = Path(record["image_path"])
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
raw_calib = read_raw_calib_from_label(image_path, Path(str(record["label_path"])))
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
roi_image = prepared.image.copy()
|
||
panel_size = (roi_image.shape[1], roi_image.shape[0])
|
||
|
||
panel_2d = draw_box_with_label(roi_image, record["gt_bbox_xyxy"], (0, 200, 0), "GT")
|
||
panel_2d = draw_box_with_label(panel_2d, record["pred_bbox_xyxy"], (0, 0, 255), "Pred")
|
||
panel_2d = annotate_panel_title(panel_2d, f"{bundle.spec.name} 2D {category}")
|
||
|
||
panel_gt = make_3d_panel(
|
||
roi_image,
|
||
prepared.calib,
|
||
record["gt_corners"],
|
||
record["gt_dims"],
|
||
record["gt_yaw_rad"],
|
||
"GT 3D",
|
||
visible_face_type=record.get("gt_visible_face_type"),
|
||
face_center=record.get("gt_face_center"),
|
||
visible_face_types=record.get("gt_visible_face_types") or parse_visible_face_names(record.get("visible_faces")),
|
||
face_center_2d=record.get("gt_face_center_2d"),
|
||
face_color=record.get("gt_face_color"),
|
||
edge_points_2d=record.get("gt_edge_points_2d"),
|
||
)
|
||
panel_pred = make_3d_panel(
|
||
roi_image,
|
||
prepared.calib,
|
||
record["pred_corners"],
|
||
record["pred_dims"],
|
||
record["pred_yaw_rad"],
|
||
"Pred 3D",
|
||
visible_face_type=record.get("pred_visible_face_type"),
|
||
face_center=record.get("pred_face_center"),
|
||
visible_face_types=record.get("pred_visible_face_types")
|
||
or ([int(record["pred_visible_face_type"])] if record.get("pred_visible_face_type") is not None else []),
|
||
face_center_2d=record.get("pred_face_center_2d"),
|
||
face_color=record.get("pred_face_color"),
|
||
edge_points_2d=record.get("pred_edge_points_2d"),
|
||
)
|
||
edge_panel_corners = record.get("pred_edge_corners")
|
||
edge_panel_dims = record.get("pred_edge_dims")
|
||
edge_panel_visible_face_type = record.get("pred_edge_visible_face_type")
|
||
edge_panel_visible_face_types = record.get("pred_edge_visible_face_types")
|
||
edge_panel_points_2d = record.get("pred_edge_edge_points_2d")
|
||
panel_edge = make_3d_panel(
|
||
roi_image,
|
||
prepared.calib,
|
||
edge_panel_corners if edge_panel_corners is not None else record["pred_corners"],
|
||
edge_panel_dims if edge_panel_dims is not None else record["pred_dims"],
|
||
record["pred_edge_yaw_rad"],
|
||
"Pred 3D EdgeYaw",
|
||
visible_face_type=(
|
||
edge_panel_visible_face_type
|
||
if edge_panel_visible_face_type is not None
|
||
else record.get("pred_visible_face_type")
|
||
),
|
||
face_center=None if edge_panel_corners is not None else record.get("pred_face_center"),
|
||
visible_face_types=(
|
||
edge_panel_visible_face_types
|
||
or record.get("pred_visible_face_types")
|
||
or ([int(record["pred_visible_face_type"])] if record.get("pred_visible_face_type") is not None else [])
|
||
),
|
||
face_center_2d=None if edge_panel_corners is not None else record.get("pred_face_center_2d"),
|
||
face_color=None if edge_panel_corners is not None else record.get("pred_face_color"),
|
||
edge_points_2d=edge_panel_points_2d if edge_panel_points_2d is not None else record.get("pred_edge_points_2d"),
|
||
rebuild=edge_panel_corners is None,
|
||
)
|
||
gt_bev = []
|
||
pred_bev = []
|
||
if record["gt_yaw_rad"] is not None:
|
||
gt_center_for_bev = np.asarray(record["gt_center"], dtype=np.float32)
|
||
if record.get("gt_visible_face_type") is not None and record.get("gt_face_center") is not None:
|
||
gt_corners = rebuild_box_corners_for_visualization(
|
||
np.asarray(record["gt_corners"], dtype=np.float32),
|
||
np.asarray(record["gt_dims"], dtype=np.float32),
|
||
float(record["gt_yaw_rad"]),
|
||
visible_face_type=int(record["gt_visible_face_type"]),
|
||
face_center_3d=np.asarray(record["gt_face_center"], dtype=np.float32),
|
||
)
|
||
if gt_corners is not None:
|
||
gt_center_for_bev = np.asarray(gt_corners, dtype=np.float32).mean(axis=0)
|
||
gt_bev.append(
|
||
{
|
||
"center": gt_center_for_bev,
|
||
"dims": np.asarray(record["gt_dims"], dtype=np.float32),
|
||
"yaw": record["gt_yaw_rad"],
|
||
}
|
||
)
|
||
if record["pred_yaw_rad"] is not None:
|
||
pred_center_for_bev = np.asarray(record["pred_center"], dtype=np.float32)
|
||
if record.get("pred_visible_face_type") is not None and record.get("pred_face_center") is not None:
|
||
pred_corners = rebuild_box_corners_for_visualization(
|
||
np.asarray(record["pred_corners"], dtype=np.float32),
|
||
np.asarray(record["pred_dims"], dtype=np.float32),
|
||
float(record["pred_yaw_rad"]),
|
||
visible_face_type=int(record["pred_visible_face_type"]),
|
||
face_center_3d=np.asarray(record["pred_face_center"], dtype=np.float32),
|
||
)
|
||
if pred_corners is not None:
|
||
pred_center_for_bev = np.asarray(pred_corners, dtype=np.float32).mean(axis=0)
|
||
pred_bev.append(
|
||
{
|
||
"center": pred_center_for_bev,
|
||
"dims": np.asarray(record["pred_dims"], dtype=np.float32),
|
||
"yaw": record["pred_yaw_rad"],
|
||
}
|
||
)
|
||
bev_rgb = create_bev_image(
|
||
gt_bev,
|
||
pred_bev,
|
||
max_range=max(80, int(max(record.get("gt_depth_m") or 0.0, record.get("pred_depth_m") or 0.0) + 20)),
|
||
lateral_range=40,
|
||
)
|
||
panel_bev = annotate_panel_title(cv2.cvtColor(bev_rgb, cv2.COLOR_RGB2BGR), "BEV GT vs Pred")
|
||
panel_text = make_text_panel(roi_image.shape, f"{bundle.spec.name} {category} #{rank}", record)
|
||
grid = assemble_visual_grid([panel_2d, panel_gt, panel_pred, panel_edge, panel_bev, panel_text], panel_size)
|
||
|
||
filename = (
|
||
f"{rank:03d}_{sanitize_name(Path(record['frame_name']).stem)}_{sanitize_name(record['cls_name'])}"
|
||
f"_g{record['gt_index']}_p{record['pred_index']}.jpg"
|
||
)
|
||
image_out = output_dir / filename
|
||
cv2.imwrite(str(image_out), grid)
|
||
manifest.append({**record, "visualization": str(image_out)})
|
||
|
||
with (output_dir / "manifest.json").open("w", encoding="utf-8") as file:
|
||
json.dump(manifest, file, indent=2, ensure_ascii=False)
|
||
return manifest
|
||
|
||
|
||
def save_class_badcase_visuals(
|
||
class_records: dict[tuple[int, str], list[dict[str, Any]]],
|
||
output_root: Path,
|
||
category: str,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_root.mkdir(parents=True, exist_ok=True)
|
||
manifests: list[dict[str, Any]] = []
|
||
for (cls_id, cls_name), records in sorted(class_records.items(), key=lambda item: item[0][0]):
|
||
if not records:
|
||
continue
|
||
class_dir = output_root / f"{int(cls_id):02d}_{sanitize_name(cls_name)}"
|
||
manifest = save_badcase_visuals(records, class_dir, category, bundle, image_root)
|
||
manifests.append(
|
||
{
|
||
"cls_id": int(cls_id),
|
||
"cls_name": str(cls_name),
|
||
"count": len(manifest),
|
||
"manifest_path": str(class_dir / "manifest.json"),
|
||
}
|
||
)
|
||
return manifests
|
||
|
||
|
||
def save_interval_badcase_visuals(
|
||
interval_records: dict[float, list[dict[str, Any]]],
|
||
frame_records: dict[str, list[dict[str, Any]]],
|
||
output_root: Path,
|
||
category: str,
|
||
bin_width: float,
|
||
unit: str,
|
||
samples_per_bin: int,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_root.mkdir(parents=True, exist_ok=True)
|
||
manifests: list[dict[str, Any]] = []
|
||
for start, records in sorted(interval_records.items(), key=lambda item: item[0], reverse=True):
|
||
if not records:
|
||
continue
|
||
bin_dir = output_root / interval_slug(float(start), float(bin_width), unit)
|
||
bin_dir.mkdir(parents=True, exist_ok=True)
|
||
label = interval_label(float(start), float(bin_width), unit=unit)
|
||
frame_candidates: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||
for record in records:
|
||
frame_candidates[str(record["image_path"])].append(record)
|
||
|
||
ranked_frames = []
|
||
for frame_key in frame_candidates:
|
||
selected_records = [
|
||
record
|
||
for record in frame_records.get(frame_key, [])
|
||
if (metric_value_for_category(record, category) is not None)
|
||
and float(metric_value_for_category(record, category)) >= float(start)
|
||
]
|
||
if not selected_records:
|
||
continue
|
||
selected_records = sorted(
|
||
selected_records,
|
||
key=lambda record: (
|
||
float(metric_value_for_category(record, category) or -float("inf")),
|
||
float(to_float(record.get("confidence")) or -float("inf")),
|
||
),
|
||
reverse=True,
|
||
)
|
||
ranked_frames.append(
|
||
(
|
||
float(metric_value_for_category(selected_records[0], category) or -float("inf")),
|
||
len(selected_records),
|
||
frame_key,
|
||
selected_records,
|
||
)
|
||
)
|
||
ranked_frames.sort(key=lambda item: (item[0], item[1], item[2]), reverse=True)
|
||
|
||
manifest = []
|
||
for rank, (_, _, frame_key, selected_records) in enumerate(ranked_frames[: max(1, int(samples_per_bin))], start=1):
|
||
image_path = Path(frame_key)
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
raw_calib = read_raw_calib_from_label(image_path, Path(str(selected_records[0]["label_path"])))
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
roi_image = prepared.image.copy()
|
||
panel_size = (roi_image.shape[1], roi_image.shape[0])
|
||
|
||
panel_2d = roi_image.copy()
|
||
panel_gt = roi_image.copy()
|
||
panel_pred = roi_image.copy()
|
||
panel_edge = roi_image.copy()
|
||
gt_bev = []
|
||
pred_bev = []
|
||
|
||
for obj_index, record in enumerate(selected_records, start=1):
|
||
panel_2d = draw_box_with_label(panel_2d, record["gt_bbox_xyxy"], (0, 200, 0), f"G{obj_index}")
|
||
panel_2d = draw_box_with_label(panel_2d, record["pred_bbox_xyxy"], (0, 0, 255), f"P{obj_index}")
|
||
draw_3d_panel_object(
|
||
panel_gt,
|
||
prepared.calib,
|
||
record["gt_corners"],
|
||
record["gt_dims"],
|
||
record["gt_yaw_rad"],
|
||
visible_face_type=record.get("gt_visible_face_type"),
|
||
face_center=record.get("gt_face_center"),
|
||
visible_face_types=record.get("gt_visible_face_types") or parse_visible_face_names(record.get("visible_faces")),
|
||
face_center_2d=record.get("gt_face_center_2d"),
|
||
face_color=record.get("gt_face_color"),
|
||
edge_points_2d=record.get("gt_edge_points_2d"),
|
||
thickness=2,
|
||
)
|
||
draw_3d_panel_object(
|
||
panel_pred,
|
||
prepared.calib,
|
||
record["pred_corners"],
|
||
record["pred_dims"],
|
||
record["pred_yaw_rad"],
|
||
visible_face_type=record.get("pred_visible_face_type"),
|
||
face_center=record.get("pred_face_center"),
|
||
visible_face_types=record.get("pred_visible_face_types")
|
||
or ([int(record["pred_visible_face_type"])] if record.get("pred_visible_face_type") is not None else []),
|
||
face_center_2d=record.get("pred_face_center_2d"),
|
||
face_color=record.get("pred_face_color"),
|
||
edge_points_2d=record.get("pred_edge_points_2d"),
|
||
thickness=2,
|
||
)
|
||
draw_3d_panel_object(
|
||
panel_edge,
|
||
prepared.calib,
|
||
record["pred_corners"],
|
||
record["pred_dims"],
|
||
record["pred_edge_yaw_rad"],
|
||
visible_face_type=record.get("pred_visible_face_type"),
|
||
face_center=record.get("pred_face_center"),
|
||
visible_face_types=record.get("pred_visible_face_types")
|
||
or ([int(record["pred_visible_face_type"])] if record.get("pred_visible_face_type") is not None else []),
|
||
face_center_2d=record.get("pred_face_center_2d"),
|
||
face_color=record.get("pred_face_color"),
|
||
edge_points_2d=record.get("pred_edge_points_2d"),
|
||
rebuild=True,
|
||
thickness=2,
|
||
)
|
||
|
||
if record["gt_yaw_rad"] is not None:
|
||
gt_center_for_bev = np.asarray(record["gt_center"], dtype=np.float32)
|
||
if record.get("gt_visible_face_type") is not None and record.get("gt_face_center") is not None:
|
||
gt_corners = rebuild_box_corners_for_visualization(
|
||
np.asarray(record["gt_corners"], dtype=np.float32),
|
||
np.asarray(record["gt_dims"], dtype=np.float32),
|
||
float(record["gt_yaw_rad"]),
|
||
visible_face_type=int(record["gt_visible_face_type"]),
|
||
face_center_3d=np.asarray(record["gt_face_center"], dtype=np.float32),
|
||
)
|
||
if gt_corners is not None:
|
||
gt_center_for_bev = np.asarray(gt_corners, dtype=np.float32).mean(axis=0)
|
||
gt_bev.append(
|
||
{
|
||
"center": gt_center_for_bev,
|
||
"dims": np.asarray(record["gt_dims"], dtype=np.float32),
|
||
"yaw": record["gt_yaw_rad"],
|
||
}
|
||
)
|
||
if record["pred_yaw_rad"] is not None:
|
||
pred_center_for_bev = np.asarray(record["pred_center"], dtype=np.float32)
|
||
if record.get("pred_visible_face_type") is not None and record.get("pred_face_center") is not None:
|
||
pred_corners = rebuild_box_corners_for_visualization(
|
||
np.asarray(record["pred_corners"], dtype=np.float32),
|
||
np.asarray(record["pred_dims"], dtype=np.float32),
|
||
float(record["pred_yaw_rad"]),
|
||
visible_face_type=int(record["pred_visible_face_type"]),
|
||
face_center_3d=np.asarray(record["pred_face_center"], dtype=np.float32),
|
||
)
|
||
if pred_corners is not None:
|
||
pred_center_for_bev = np.asarray(pred_corners, dtype=np.float32).mean(axis=0)
|
||
pred_bev.append(
|
||
{
|
||
"center": pred_center_for_bev,
|
||
"dims": np.asarray(record["pred_dims"], dtype=np.float32),
|
||
"yaw": record["pred_yaw_rad"],
|
||
}
|
||
)
|
||
|
||
panel_2d = annotate_panel_title(panel_2d, f"{bundle.spec.name} 2D {category}_{label}")
|
||
panel_gt = annotate_panel_title(panel_gt, "GT 3D")
|
||
panel_pred = annotate_panel_title(panel_pred, "Pred 3D")
|
||
panel_edge = annotate_panel_title(panel_edge, "Pred 3D EdgeYaw")
|
||
bev_rgb = create_bev_image(
|
||
gt_bev,
|
||
pred_bev,
|
||
max_range=max(
|
||
80,
|
||
int(
|
||
max(
|
||
[to_float(record.get("gt_depth_m")) or 0.0 for record in selected_records]
|
||
+ [to_float(record.get("pred_depth_m")) or 0.0 for record in selected_records]
|
||
)
|
||
+ 20
|
||
),
|
||
),
|
||
lateral_range=40,
|
||
)
|
||
panel_bev = annotate_panel_title(cv2.cvtColor(bev_rgb, cv2.COLOR_RGB2BGR), "BEV GT vs Pred")
|
||
panel_text = make_interval_bin_text_panel(
|
||
roi_image.shape,
|
||
f"{bundle.spec.name} {category} {label} #{rank}",
|
||
category,
|
||
float(start),
|
||
label,
|
||
selected_records,
|
||
)
|
||
grid = assemble_visual_grid([panel_2d, panel_gt, panel_pred, panel_edge, panel_bev, panel_text], panel_size)
|
||
|
||
metric_values = [metric_value_for_category(record, category) for record in selected_records]
|
||
metric_values = [float(value) for value in metric_values if value is not None]
|
||
image_out = bin_dir / (
|
||
f"{rank:03d}_{sanitize_name(image_path.stem)}_{sanitize_name(category)}_{sanitize_name(label)}_n{len(selected_records)}.jpg"
|
||
)
|
||
cv2.imwrite(str(image_out), grid)
|
||
manifest.append(
|
||
{
|
||
"roi": bundle.spec.name,
|
||
"frame_name": image_path.name,
|
||
"image_path": str(image_path),
|
||
"bin_start": float(start),
|
||
"bin_end": float(start + bin_width),
|
||
"bin_label": label,
|
||
"badcase_count": int(len(selected_records)),
|
||
"max_metric_value": max(metric_values) if metric_values else None,
|
||
"mean_metric_value": mean_or_none(metric_values),
|
||
"metric_unit": unit,
|
||
"cls_summary": ", ".join(sorted({str(record.get("cls_name", "unknown")) for record in selected_records})[:6]),
|
||
"visualization": str(image_out),
|
||
}
|
||
)
|
||
|
||
with (bin_dir / "manifest.json").open("w", encoding="utf-8") as file:
|
||
json.dump(manifest, file, indent=2, ensure_ascii=False)
|
||
manifests.append(
|
||
{
|
||
"bin_start": float(start),
|
||
"bin_end": float(start + bin_width),
|
||
"bin_label": label,
|
||
"count": len(manifest),
|
||
"manifest_path": str(bin_dir / "manifest.json"),
|
||
}
|
||
)
|
||
return manifests
|
||
|
||
|
||
def save_2d_badcase_visuals(
|
||
records: list[dict[str, Any]],
|
||
output_dir: Path,
|
||
category: str,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest = []
|
||
for rank, record in enumerate(records, start=1):
|
||
image_path = Path(record["image_path"])
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
raw_calib = read_raw_calib_from_label(image_path, Path(str(record["label_path"])))
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
roi_image = prepared.image.copy()
|
||
box_color = (0, 200, 0) if record["kind"] == "false_negative" else (0, 0, 255)
|
||
box_label = "GT miss" if record["kind"] == "false_negative" else "Pred fp"
|
||
panel_2d = draw_box_with_label(roi_image, record["bbox_xyxy"], box_color, box_label)
|
||
panel_2d = annotate_panel_title(panel_2d, f"{bundle.spec.name} 2D {category}")
|
||
text_panel = make_2d_badcase_text_panel(roi_image.shape, f"{bundle.spec.name} {category} #{rank}", record)
|
||
grid = np.concatenate([panel_2d, text_panel], axis=1)
|
||
|
||
filename = (
|
||
f"{rank:03d}_{sanitize_name(Path(record['frame_name']).stem)}_{sanitize_name(record['cls_name'])}"
|
||
f"_{sanitize_name(record['kind'])}_g{record.get('gt_index')}_p{record.get('pred_index')}.jpg"
|
||
)
|
||
image_out = output_dir / filename
|
||
cv2.imwrite(str(image_out), grid)
|
||
manifest.append({**record, "visualization": str(image_out)})
|
||
|
||
with (output_dir / "manifest.json").open("w", encoding="utf-8") as file:
|
||
json.dump(manifest, file, indent=2, ensure_ascii=False)
|
||
return manifest
|
||
|
||
|
||
def save_face_selection_badcase_visuals(
|
||
records: list[dict[str, Any]],
|
||
output_dir: Path,
|
||
category: str,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest = []
|
||
for rank, record in enumerate(records, start=1):
|
||
image_path = Path(record["image_path"])
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
raw_calib = read_raw_calib_from_label(image_path, Path(str(record["label_path"])))
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
roi_image = prepared.image.copy()
|
||
panel_2d = draw_box_with_label(
|
||
roi_image,
|
||
record["gt_bbox_xyxy"],
|
||
(0, 200, 0),
|
||
f"GT {record.get('gt_face_selection_label', '')}",
|
||
)
|
||
panel_2d = draw_box_with_label(
|
||
panel_2d,
|
||
record["pred_bbox_xyxy"],
|
||
(0, 0, 255),
|
||
f"Pred {record.get('pred_face_selection_label', '')}",
|
||
)
|
||
panel_2d = annotate_panel_title(panel_2d, f"{bundle.spec.name} {category}")
|
||
text_panel = make_face_selection_text_panel(roi_image.shape, f"{bundle.spec.name} {category} #{rank}", record)
|
||
grid = np.concatenate([panel_2d, text_panel], axis=1)
|
||
|
||
filename = (
|
||
f"{rank:03d}_{sanitize_name(Path(record['frame_name']).stem)}"
|
||
f"_g{record.get('gt_index')}_p{record.get('pred_index')}.jpg"
|
||
)
|
||
image_out = output_dir / filename
|
||
cv2.imwrite(str(image_out), grid)
|
||
manifest.append({**record, "visualization": str(image_out)})
|
||
|
||
with (output_dir / "manifest.json").open("w", encoding="utf-8") as file:
|
||
json.dump(manifest, file, indent=2, ensure_ascii=False)
|
||
return manifest
|
||
|
||
|
||
def save_fake_class_badcase_visuals(
|
||
records: list[dict[str, Any]],
|
||
output_dir: Path,
|
||
category: str,
|
||
bundle,
|
||
image_root: Path,
|
||
) -> list[dict[str, Any]]:
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest = []
|
||
for rank, record in enumerate(records, start=1):
|
||
image_path = Path(record["image_path"])
|
||
image_bgr = cv2.imread(str(image_path), cv2.IMREAD_COLOR)
|
||
if image_bgr is None:
|
||
continue
|
||
raw_calib = read_raw_calib_from_label(image_path, Path(str(record["label_path"])))
|
||
prepared = prepare_roi_image(image_bgr, raw_calib, bundle.spec, bundle.imgsz)
|
||
roi_image = prepared.image.copy()
|
||
panel_2d = draw_box_with_label(
|
||
roi_image,
|
||
record["gt_bbox_xyxy"],
|
||
(0, 200, 0),
|
||
f"GT {record.get('gt_fake_class_label', record.get('gt_cls_name', ''))}",
|
||
dashed=record.get("gt_occlusion_label") == "occluded",
|
||
)
|
||
panel_2d = draw_box_with_label(
|
||
panel_2d,
|
||
record["pred_bbox_xyxy"],
|
||
(0, 0, 255),
|
||
f"Pred {record.get('pred_fake_class_label', record.get('pred_cls_name', ''))}",
|
||
dashed=record.get("pred_occlusion_label") == "occluded",
|
||
)
|
||
panel_2d = annotate_panel_title(panel_2d, f"{bundle.spec.name} fake_class {category}")
|
||
text_panel = make_fake_class_text_panel(roi_image.shape, f"{bundle.spec.name} {category} #{rank}", record)
|
||
grid = np.concatenate([panel_2d, text_panel], axis=1)
|
||
|
||
filename = (
|
||
f"{rank:03d}_{sanitize_name(Path(record['frame_name']).stem)}"
|
||
f"_{sanitize_name(record['kind'])}"
|
||
f"_g{record.get('gt_index')}_p{record.get('pred_index')}.jpg"
|
||
)
|
||
image_out = output_dir / filename
|
||
cv2.imwrite(str(image_out), grid)
|
||
manifest.append({**record, "visualization": str(image_out)})
|
||
|
||
with (output_dir / "manifest.json").open("w", encoding="utf-8") as file:
|
||
json.dump(manifest, file, indent=2, ensure_ascii=False)
|
||
return manifest
|
||
|
||
|
||
def build_thresholded_2d_artifacts(
|
||
eval_packets: list[dict[str, Any]],
|
||
roi_name: str,
|
||
names_dict: dict[int, str],
|
||
confidence_threshold: float,
|
||
topk_badcases: int,
|
||
badcase_random_seed: int,
|
||
roi_output: Path,
|
||
bundle,
|
||
image_root: Path,
|
||
max_abs_lateral_m: float = FOCUSED_CONFUSION_MAX_ABS_LATERAL_M,
|
||
max_abs_longitudinal_m: float = FOCUSED_CONFUSION_MAX_ABS_LONGITUDINAL_M,
|
||
required_difficulty: int = FOCUSED_CONFUSION_REQUIRED_DIFFICULTY,
|
||
) -> dict[str, Any]:
|
||
bad_2d_fn_handle, bad_2d_fn_writer = make_writer(roi_output / "bad_cases_2d_false_negative.csv", fieldnames=BADCASE_2D_FIELDS)
|
||
bad_2d_fp_handle, bad_2d_fp_writer = make_writer(roi_output / "bad_cases_2d_false_positive.csv", fieldnames=BADCASE_2D_FIELDS)
|
||
fake_class_handle, fake_class_writer = make_writer(roi_output / "bad_cases_fake_class.csv", fieldnames=FAKE_CLASS_BADCASE_FIELDS)
|
||
|
||
class_counts = defaultdict(lambda: {"gt_total": 0, "pred_total": 0, "matched_2d": 0})
|
||
overall_counts = {"gt_total": 0, "pred_total": 0, "matched_2d": 0}
|
||
fake_class_store_metrics = make_label_accuracy_store(FAKE_CLASS_LABEL_ORDER)
|
||
fake_class_counts = {"fake_vs_non_fake": 0, "fake_internal_confusion": 0, "visible_internal_confusion": 0}
|
||
confusion_matrix_2d = ConfusionMatrix(names=names_dict, task="detect")
|
||
focused_confusion_matrix_2d = ConfusionMatrix(names=names_dict, task="detect")
|
||
focused_confusion_gt_total = 0
|
||
focused_confusion_pred_spatial_total = 0
|
||
|
||
rng = random.Random(int(badcase_random_seed) + (2000 if str(roi_name).lower() == "roi1" else 1500))
|
||
bad_2d_fn_store = make_reservoir_store()
|
||
bad_2d_fp_store = make_reservoir_store()
|
||
fake_class_store = make_reservoir_store()
|
||
|
||
for packet in eval_packets:
|
||
gt_cls = np.asarray(packet["gt_cls"], dtype=np.int32).reshape(-1)
|
||
gt_boxes = np.asarray(packet["gt_boxes"], dtype=np.float32).reshape(-1, 4)
|
||
pred_cls_all = np.asarray(packet["pred_cls"], dtype=np.int32).reshape(-1)
|
||
pred_boxes_all = np.asarray(packet["pred_boxes"], dtype=np.float32).reshape(-1, 4)
|
||
pred_conf_all = np.asarray(packet["pred_conf"], dtype=np.float32).reshape(-1)
|
||
gt_focus_mask = np.asarray(packet["gt_focus_mask"], dtype=bool).reshape(-1)
|
||
pred_focus_mask_all = np.asarray(packet["pred_focus_mask"], dtype=bool).reshape(-1)
|
||
|
||
keep = pred_conf_all > float(confidence_threshold) if pred_conf_all.size else np.zeros((0,), dtype=bool)
|
||
pred_cls = pred_cls_all[keep]
|
||
pred_boxes = pred_boxes_all[keep]
|
||
pred_conf = pred_conf_all[keep]
|
||
pred_focus_mask = pred_focus_mask_all[keep]
|
||
|
||
overall_counts["gt_total"] += int(len(gt_cls))
|
||
overall_counts["pred_total"] += int(len(pred_cls))
|
||
focused_confusion_gt_total += int(np.sum(gt_focus_mask))
|
||
focused_confusion_pred_spatial_total += int(np.sum(pred_focus_mask))
|
||
for cls_id in gt_cls.tolist():
|
||
class_counts[int(cls_id)]["gt_total"] += 1
|
||
for cls_id in pred_cls.tolist():
|
||
class_counts[int(cls_id)]["pred_total"] += 1
|
||
|
||
confusion_matrix_2d.process_batch(
|
||
detections={
|
||
"cls": torch.from_numpy(pred_cls_all.astype(np.float32)),
|
||
"conf": torch.from_numpy(pred_conf_all.astype(np.float32)),
|
||
"bboxes": torch.from_numpy(pred_boxes_all.astype(np.float32)),
|
||
},
|
||
batch={
|
||
"cls": torch.from_numpy(gt_cls.astype(np.float32)),
|
||
"bboxes": torch.from_numpy(gt_boxes.astype(np.float32)),
|
||
},
|
||
conf=float(confidence_threshold),
|
||
iou_thres=0.5,
|
||
)
|
||
update_subset_confusion_matrix(
|
||
focused_confusion_matrix_2d,
|
||
gt_cls=gt_cls,
|
||
gt_boxes=gt_boxes,
|
||
pred_cls=pred_cls_all,
|
||
pred_boxes=pred_boxes_all,
|
||
pred_conf=pred_conf_all,
|
||
gt_mask=gt_focus_mask,
|
||
pred_mask=pred_focus_mask_all,
|
||
conf=float(confidence_threshold),
|
||
iou_thres=0.5,
|
||
)
|
||
|
||
matches, iou_matrix = greedy_match_indices(gt_cls, gt_boxes, pred_cls, pred_boxes, iou_thr=0.5)
|
||
fake_matches, fake_iou_matrix = greedy_match_indices_any_class(gt_boxes, pred_boxes, iou_thr=0.5)
|
||
overall_counts["matched_2d"] += int(len(matches))
|
||
matched_gt = set(matches[:, 0].tolist()) if len(matches) else set()
|
||
matched_pred = set(matches[:, 1].tolist()) if len(matches) else set()
|
||
|
||
image_path = Path(str(packet["image_path"]))
|
||
label_path = Path(str(packet["label_path"]))
|
||
sample_index = int(packet["sample_index"])
|
||
|
||
for gt_index, pred_index in matches.tolist() if len(matches) else []:
|
||
class_counts[int(gt_cls[int(gt_index)])]["matched_2d"] += 1
|
||
|
||
for gt_index, pred_index in fake_matches.tolist() if len(fake_matches) else []:
|
||
gt_ci = int(gt_cls[int(gt_index)])
|
||
pred_ci = int(pred_cls[int(pred_index)])
|
||
gt_cn = get_cls_name(names_dict, gt_ci)
|
||
pred_cn = get_cls_name(names_dict, pred_ci)
|
||
gt_fake_label = fake_class_label(gt_cn)
|
||
pred_fake_label = fake_class_label(pred_cn)
|
||
add_label_accuracy_sample(fake_class_store_metrics, gt_fake_label, pred_fake_label)
|
||
if gt_fake_label == pred_fake_label:
|
||
continue
|
||
match_iou = float(fake_iou_matrix[int(gt_index), int(pred_index)]) if fake_iou_matrix.size else 0.0
|
||
fake_record = make_fake_class_badcase_record(
|
||
sample_index=sample_index,
|
||
roi_name=roi_name,
|
||
image_path=image_path,
|
||
label_path=label_path,
|
||
gt_index=int(gt_index),
|
||
pred_index=int(pred_index),
|
||
gt_cls_id=gt_ci,
|
||
gt_cls_name=gt_cn,
|
||
pred_cls_id=pred_ci,
|
||
pred_cls_name=pred_cn,
|
||
gt_bbox_xyxy=gt_boxes[int(gt_index)],
|
||
pred_bbox_xyxy=pred_boxes[int(pred_index)],
|
||
match_iou=match_iou,
|
||
confidence=float(pred_conf[int(pred_index)]),
|
||
)
|
||
fake_class_writer.writerow(fake_class_record_to_csv_row(fake_record))
|
||
reservoir_add(fake_class_store, fake_record, topk_badcases, rng)
|
||
fake_class_counts[fake_record["kind"]] += 1
|
||
|
||
for gt_index, gt_box in enumerate(gt_boxes):
|
||
if gt_index in matched_gt:
|
||
continue
|
||
cls_id = int(gt_cls[gt_index])
|
||
cls_name = get_cls_name(names_dict, cls_id)
|
||
iou_row = iou_matrix[gt_index] if iou_matrix.shape[1] > 0 else np.zeros((0,), dtype=np.float32)
|
||
same_class_mask = pred_cls == cls_id if len(pred_cls) else np.zeros((0,), dtype=bool)
|
||
record_2d = make_2d_badcase_record(
|
||
sample_index=sample_index,
|
||
roi_name=roi_name,
|
||
image_path=image_path,
|
||
label_path=label_path,
|
||
kind="false_negative",
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
bbox_xyxy=gt_box,
|
||
gt_index=gt_index,
|
||
pred_index=None,
|
||
confidence=None,
|
||
max_iou_any=float(np.max(iou_row)) if iou_row.size else 0.0,
|
||
max_iou_same_class=float(np.max(iou_row[same_class_mask])) if same_class_mask.any() else 0.0,
|
||
)
|
||
bad_2d_fn_writer.writerow(record_2d_to_csv_row(record_2d))
|
||
reservoir_add(bad_2d_fn_store, record_2d, topk_badcases, rng)
|
||
|
||
for pred_index, pred_box in enumerate(pred_boxes):
|
||
if pred_index in matched_pred:
|
||
continue
|
||
cls_id = int(pred_cls[pred_index])
|
||
cls_name = get_cls_name(names_dict, cls_id)
|
||
iou_col = iou_matrix[:, pred_index] if iou_matrix.shape[0] > 0 else np.zeros((0,), dtype=np.float32)
|
||
same_class_mask = gt_cls == cls_id if len(gt_cls) else np.zeros((0,), dtype=bool)
|
||
record_2d = make_2d_badcase_record(
|
||
sample_index=sample_index,
|
||
roi_name=roi_name,
|
||
image_path=image_path,
|
||
label_path=label_path,
|
||
kind="false_positive",
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
bbox_xyxy=pred_box,
|
||
gt_index=None,
|
||
pred_index=pred_index,
|
||
confidence=float(pred_conf[pred_index]),
|
||
max_iou_any=float(np.max(iou_col)) if iou_col.size else 0.0,
|
||
max_iou_same_class=float(np.max(iou_col[same_class_mask])) if same_class_mask.any() else 0.0,
|
||
)
|
||
bad_2d_fp_writer.writerow(record_2d_to_csv_row(record_2d))
|
||
reservoir_add(bad_2d_fp_store, record_2d, topk_badcases, rng)
|
||
|
||
bad_2d_fn_handle.close()
|
||
bad_2d_fp_handle.close()
|
||
|
||
class_rows = {}
|
||
for cls_id, counts in class_counts.items():
|
||
precision = rate_or_none(int(counts["matched_2d"]), int(counts["pred_total"]))
|
||
recall = rate_or_none(int(counts["matched_2d"]), int(counts["gt_total"]))
|
||
class_rows[int(cls_id)] = {
|
||
"gt_total": int(counts["gt_total"]),
|
||
"pred_total": int(counts["pred_total"]),
|
||
"matched_2d": int(counts["matched_2d"]),
|
||
"precision_2d": precision,
|
||
"recall_2d": recall,
|
||
"f1_2d": f1_or_none(precision, recall),
|
||
"false_negatives_2d": max(0, int(counts["gt_total"]) - int(counts["matched_2d"])),
|
||
"false_positives_2d": max(0, int(counts["pred_total"]) - int(counts["matched_2d"])),
|
||
}
|
||
|
||
overall_precision = rate_or_none(int(overall_counts["matched_2d"]), int(overall_counts["pred_total"]))
|
||
overall_recall = rate_or_none(int(overall_counts["matched_2d"]), int(overall_counts["gt_total"]))
|
||
overall_row = {
|
||
"gt_total": int(overall_counts["gt_total"]),
|
||
"pred_total": int(overall_counts["pred_total"]),
|
||
"matched_2d": int(overall_counts["matched_2d"]),
|
||
"precision_2d": overall_precision,
|
||
"recall_2d": overall_recall,
|
||
"f1_2d": f1_or_none(overall_precision, overall_recall),
|
||
"false_negatives_2d": max(0, int(overall_counts["gt_total"]) - int(overall_counts["matched_2d"])),
|
||
"false_positives_2d": max(0, int(overall_counts["pred_total"]) - int(overall_counts["matched_2d"])),
|
||
}
|
||
|
||
classification_summary_2d = summarize_2d_classification_from_confusion(confusion_matrix_2d)
|
||
focused_classification_summary_2d = summarize_2d_classification_from_confusion(focused_confusion_matrix_2d)
|
||
fake_class_summary = summarize_label_accuracy_store(fake_class_store_metrics)
|
||
confusion_matrix_plot_path = roi_output / "confusion_matrix_normalized.png"
|
||
confusion_matrix_2d.plot(normalize=True, save_dir=str(roi_output))
|
||
confusion_matrix_plot = str(confusion_matrix_plot_path) if confusion_matrix_plot_path.is_file() else None
|
||
focused_confusion_output = roi_output / "confusion_matrix_easy_near_no_occlusion"
|
||
focused_confusion_output.mkdir(parents=True, exist_ok=True)
|
||
focused_confusion_matrix_plot_path = focused_confusion_output / "confusion_matrix_normalized.png"
|
||
focused_confusion_matrix_2d.plot(normalize=True, save_dir=str(focused_confusion_output))
|
||
focused_confusion_plot = str(focused_confusion_matrix_plot_path) if focused_confusion_matrix_plot_path.is_file() else None
|
||
|
||
bad_2d_fn_records = reservoir_records(bad_2d_fn_store, rng)
|
||
bad_2d_fp_records = reservoir_records(bad_2d_fp_store, rng)
|
||
fake_class_records = reservoir_records(fake_class_store, rng)
|
||
bad_2d_fn_manifest = save_2d_badcase_visuals(
|
||
bad_2d_fn_records,
|
||
roi_output / "visuals" / "2d_false_negative",
|
||
"2d_false_negative",
|
||
bundle,
|
||
image_root,
|
||
)
|
||
bad_2d_fp_manifest = save_2d_badcase_visuals(
|
||
bad_2d_fp_records,
|
||
roi_output / "visuals" / "2d_false_positive",
|
||
"2d_false_positive",
|
||
bundle,
|
||
image_root,
|
||
)
|
||
fake_class_manifest = save_fake_class_badcase_visuals(
|
||
fake_class_records,
|
||
roi_output / "visuals" / "fake_class",
|
||
"fake_class",
|
||
bundle,
|
||
image_root,
|
||
)
|
||
|
||
fake_class_handle.close()
|
||
|
||
return {
|
||
"overall": overall_row,
|
||
"per_class": class_rows,
|
||
"classification_summary_2d": classification_summary_2d,
|
||
"focused_classification_summary_2d": focused_classification_summary_2d,
|
||
"confusion_matrix_plot_path_2d": confusion_matrix_plot,
|
||
"focused_confusion_matrix_plot_path_2d": focused_confusion_plot,
|
||
"focused_confusion_filter": {
|
||
"max_abs_lateral_m": float(max_abs_lateral_m),
|
||
"max_abs_longitudinal_m": float(max_abs_longitudinal_m),
|
||
"required_difficulty": int(required_difficulty),
|
||
"confidence_threshold": float(confidence_threshold),
|
||
"gt_count": int(focused_confusion_gt_total),
|
||
"pred_spatial_count": int(focused_confusion_pred_spatial_total),
|
||
},
|
||
"fake_class_summary": {
|
||
**fake_class_summary,
|
||
"fake_vs_non_fake": int(fake_class_counts["fake_vs_non_fake"]),
|
||
"fake_internal_confusion": int(fake_class_counts["fake_internal_confusion"]),
|
||
"visible_internal_confusion": int(fake_class_counts["visible_internal_confusion"]),
|
||
"saved_topk": len(fake_class_records),
|
||
},
|
||
"badcase_counts": {
|
||
"2d_false_negative_saved_topk": len(bad_2d_fn_records),
|
||
"2d_false_positive_saved_topk": len(bad_2d_fp_records),
|
||
"fake_class_saved_topk": len(fake_class_records),
|
||
},
|
||
"badcase_manifest_paths": {
|
||
"2d_false_negative": str(roi_output / "visuals" / "2d_false_negative" / "manifest.json"),
|
||
"2d_false_positive": str(roi_output / "visuals" / "2d_false_positive" / "manifest.json"),
|
||
"fake_class": str(roi_output / "visuals" / "fake_class" / "manifest.json"),
|
||
},
|
||
"badcase_manifest_sizes": {
|
||
"2d_false_negative": len(bad_2d_fn_manifest),
|
||
"2d_false_positive": len(bad_2d_fp_manifest),
|
||
"fake_class": len(fake_class_manifest),
|
||
},
|
||
}
|
||
|
||
|
||
def write_json(path: Path, payload: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with path.open("w", encoding="utf-8") as file:
|
||
json.dump(payload, file, indent=2, ensure_ascii=False)
|
||
|
||
|
||
def write_class_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
if not rows:
|
||
with path.open("w", encoding="utf-8", newline="") as file:
|
||
writer = csv.writer(file)
|
||
writer.writerow(["cls_id", "cls_name"])
|
||
return
|
||
fieldnames = list(rows[0].keys())
|
||
with path.open("w", encoding="utf-8", newline="") as file:
|
||
writer = csv.DictWriter(file, fieldnames=fieldnames)
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def write_markdown_summary(path: Path, roi_name: str, payload: dict[str, Any], model_path: str, split_file: str) -> None:
|
||
overall = payload["overall"]
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
confidence_curve_paths_2d = payload.get("confidence_curve_paths_2d") or {}
|
||
focused_confusion_filter = payload.get("focused_confusion_filter") or {}
|
||
focused_classification_summary_2d = payload.get("focused_classification_summary_2d") or {}
|
||
face_selection_summary = payload.get("face_selection_summary") or {}
|
||
occlusion_binary_summary = payload.get("occlusion_binary_summary") or {}
|
||
large_vehicle_compare = payload.get("large_vehicle_compare") or {}
|
||
large_vehicle_summary = large_vehicle_compare.get("summary") or {}
|
||
yaw_contributors = payload["top_yaw_contributors"]
|
||
horizontal = payload["top_horizontal_classes"]
|
||
vertical = payload["top_vertical_classes"]
|
||
worst_depth = payload["worst_depth_bins"]
|
||
worst_diag = payload["worst_bbox_bins"]
|
||
worst_faces = payload["worst_face_buckets"]
|
||
lines = [
|
||
f"# {roi_name.upper()} Validation Error Analysis",
|
||
"",
|
||
f"- model: `{model_path}`",
|
||
f"- split_file: `{split_file}`",
|
||
f"- matched_2d: {overall['matched_2d']}",
|
||
f"- matched_3d: {overall['matched_3d']}",
|
||
f"- yaw_mae_deg: {overall['yaw_mae_deg'] if overall['yaw_mae_deg'] is not None else 'n/a'}",
|
||
f"- x_abs_mae_m: {overall['x_abs_mae_m'] if overall['x_abs_mae_m'] is not None else 'n/a'}",
|
||
f"- z_abs_mae_m: {overall['z_abs_mae_m'] if overall['z_abs_mae_m'] is not None else 'n/a'}",
|
||
f"- configured_2d_confidence: {format_float(to_float(payload.get('configured_confidence_2d')), 3)}",
|
||
f"- recommended_2d_confidence: {format_float(to_float(threshold_advice_2d.get('recommended_confidence')), 3)}",
|
||
f"- position_error_basis: face center for face_3d classes; box center otherwise",
|
||
f"- 2d_precision@recommended_conf: {format_percent(overall.get('precision_2d'))}",
|
||
f"- 2d_recall@recommended_conf: {format_percent(overall.get('recall_2d'))}",
|
||
f"- 2d_f1@recommended_conf: {format_percent(overall.get('f1_2d'))}",
|
||
f"- mean_class_f1_for_threshold_advice: {format_percent(threshold_advice_2d.get('mean_f1'))}",
|
||
f"- focused_2d_gt_count: {int(focused_confusion_filter.get('gt_count', 0) or 0)}",
|
||
f"- focused_2d_cls_acc: {focused_classification_summary_2d.get('overall_accuracy') if focused_classification_summary_2d.get('overall_accuracy') is not None else 'n/a'}",
|
||
f"- focused_2d_longitudinal_limit_m: {format_float(to_float(focused_confusion_filter.get('max_abs_longitudinal_m')), 1)}",
|
||
f"- face_selection_acc: {format_percent(face_selection_summary.get('overall_accuracy'))}",
|
||
f"- face_selection_mean_acc: {format_percent(face_selection_summary.get('mean_accuracy'))}",
|
||
f"- occlusion_binary_acc: {format_percent(occlusion_binary_summary.get('overall_accuracy'))}",
|
||
f"- large_vehicle_scope: {large_vehicle_compare.get('class_scope', LARGE_VEHICLE_CLASS_SCOPE_TEXT)}",
|
||
f"- large_vehicle_two_face_pairs: {int(large_vehicle_summary.get('yaw_compare_count', 0) or 0)}",
|
||
f"- large_vehicle_edge_better_rate: {large_vehicle_summary.get('yaw_compare_edge_better_rate') if large_vehicle_summary.get('yaw_compare_edge_better_rate') is not None else 'n/a'}",
|
||
"",
|
||
"## 2D Confidence Advice",
|
||
"",
|
||
f"- Detection-dependent report results use `conf > {format_float(to_float(payload.get('report_confidence_2d')), 3)}`.",
|
||
f"- Threshold advice source: `{threshold_advice_2d.get('source', 'n/a')}` from the ROI F1-confidence sweep instead of the configured default `{format_float(to_float(payload.get('configured_confidence_2d')), 3)}`.",
|
||
f"- Mean class precision / recall / F1 at the advised threshold: {format_percent(threshold_advice_2d.get('mean_precision'))} / {format_percent(threshold_advice_2d.get('mean_recall'))} / {format_percent(threshold_advice_2d.get('mean_f1'))}.",
|
||
"",
|
||
"## Why Yaw Error Is Large",
|
||
"",
|
||
"Top class contributors by total yaw error:",
|
||
]
|
||
for row in yaw_contributors:
|
||
lines.append(
|
||
f"- {row['cls_name']} (id={row['cls_id']}): matched_3d={row['matched_3d']}, "
|
||
f"yaw_mae_deg={row['yaw_mae_deg']}, yaw_contribution={row['yaw_contribution_rate']}"
|
||
)
|
||
lines.extend(["", "Worst depth bins by yaw MAE:"])
|
||
for key, row in worst_depth:
|
||
lines.append(f"- {key}: matched_3d={row['matched_3d']}, yaw_mae_deg={row['yaw_mae_deg']}")
|
||
lines.extend(["", "Worst box-size bins by yaw MAE:"])
|
||
for key, row in worst_diag:
|
||
lines.append(f"- {key}: matched_3d={row['matched_3d']}, yaw_mae_deg={row['yaw_mae_deg']}")
|
||
lines.extend(["", "Worst face-visibility buckets by yaw MAE:"])
|
||
for key, row in worst_faces:
|
||
lines.append(
|
||
f"- {key}: matched_3d={row['matched_3d']}, yaw_mae_deg={row['yaw_mae_deg']}, "
|
||
f"edge_based_yaw_mae_deg={row['edge_based_yaw_mae_deg']}"
|
||
)
|
||
lines.extend(["", "## Bad Horizontal Errors (> threshold)", ""])
|
||
for row in horizontal:
|
||
lines.append(f"- {row['cls_name']} (id={row['cls_id']}): x_bad_count={row['x_bad_count']}, x_bad_rate={row['x_bad_rate']}")
|
||
lines.extend(["", "## Bad Vertical Errors (> threshold)", ""])
|
||
for row in vertical:
|
||
lines.append(f"- {row['cls_name']} (id={row['cls_id']}): z_bad_count={row['z_bad_count']}, z_bad_rate={row['z_bad_rate']}")
|
||
lines.extend(
|
||
[
|
||
"",
|
||
"## Extra Artifacts",
|
||
"",
|
||
f"- 2d_f1_curve: `{confidence_curve_paths_2d.get('f1_curve', 'n/a')}`",
|
||
f"- focused_confusion_matrix: `{payload.get('focused_confusion_matrix_plot_path_2d', 'n/a')}`",
|
||
f"- html_report: `{path.parent.parent / 'report.html'}`",
|
||
]
|
||
)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def write_markdown_summary_zh(
|
||
path: Path,
|
||
roi_name: str,
|
||
payload: dict[str, Any],
|
||
model_path: str,
|
||
split_file: str,
|
||
horizontal_csv: Path,
|
||
vertical_csv: Path,
|
||
yaw_depth_csv: Path,
|
||
yaw_heading_csv: Path,
|
||
) -> None:
|
||
overall = payload["overall"]
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
confidence_curve_paths_2d = payload.get("confidence_curve_paths_2d") or {}
|
||
focused_confusion_filter = payload.get("focused_confusion_filter") or {}
|
||
focused_classification_summary_2d = payload.get("focused_classification_summary_2d") or {}
|
||
face_selection_summary = payload.get("face_selection_summary") or {}
|
||
occlusion_binary_summary = payload.get("occlusion_binary_summary") or {}
|
||
large_vehicle_compare = payload.get("large_vehicle_compare") or {}
|
||
large_vehicle_summary = large_vehicle_compare.get("summary") or {}
|
||
per_class_insights = payload["per_class_interval_insights"]
|
||
lines = [
|
||
f"# {roi_name.upper()} 验证集误差统计分析",
|
||
"",
|
||
"## 统计口径",
|
||
"",
|
||
f"- 模型: `{model_path}`",
|
||
f"- 验证列表: `{split_file}`",
|
||
"- 匹配规则: 同类别 + IoU>=0.5,与训练期 validator 保持一致。",
|
||
"- 区间解释: 这里把“每 1m / 5m / 10m interval”理解为按 GT 前向距离 `z_gt` 分桶。",
|
||
"- 水平误差: `|pred_x - gt_x|`,按 GT 横向位置 `x_gt` 在 [-30m, 30m) 内做 5m 分桶统计。",
|
||
"- 垂直误差: `|pred_y - gt_y|`,按 5m 深度分桶统计。",
|
||
"- 偏航误差: `|yaw_pred - yaw_gt|`,按 10m 深度分桶统计均值。",
|
||
"- GT 偏航分桶: 额外提供按 GT yaw 每 10deg 分桶的偏航绝对误差与相对误差,其中相对误差按 `|yaw_err| / |gt_yaw|` 统计。",
|
||
"- 位置误差口径: `face_3d_classes` 用 face center,其他类别用 whole-box center。",
|
||
"",
|
||
"## 整体结果",
|
||
"",
|
||
f"- matched_3d: {overall['matched_3d']}",
|
||
f"- yaw_mae_deg: {format_float(overall['yaw_mae_deg'], 3)}",
|
||
f"- x_abs_mae_m: {format_float(overall['x_abs_mae_m'], 3)}",
|
||
f"- z_abs_mae_m: {format_float(overall['z_abs_mae_m'], 3)}",
|
||
f"- 当前配置 2D conf: {format_float(to_float(payload.get('configured_confidence_2d')), 3)}",
|
||
f"- 建议 2D conf: {format_float(to_float(threshold_advice_2d.get('recommended_confidence')), 3)}",
|
||
f"- 建议阈值下 2D precision / recall / F1: {format_percent(overall.get('precision_2d'))} / {format_percent(overall.get('recall_2d'))} / {format_percent(overall.get('f1_2d'))}",
|
||
f"- F1 阈值建议对应的 mean class F1: {format_percent(threshold_advice_2d.get('mean_f1'))}",
|
||
f"- 近距离无遮挡 GT 数: {int(focused_confusion_filter.get('gt_count', 0) or 0)}",
|
||
f"- 近距离无遮挡 2D 分类准确率: {format_percent(focused_classification_summary_2d.get('overall_accuracy'))}",
|
||
f"- 近距离无遮挡纵向范围: |z|<{format_float(to_float(focused_confusion_filter.get('max_abs_longitudinal_m')), 1)}m",
|
||
f"- face/cut 选择准确率: {format_percent(face_selection_summary.get('overall_accuracy'))},mean acc: {format_percent(face_selection_summary.get('mean_accuracy'))}",
|
||
f"- 遮挡二分类准确率: {format_percent(occlusion_binary_summary.get('overall_accuracy'))}",
|
||
f"- 大车对比范围: {large_vehicle_compare.get('class_scope', LARGE_VEHICLE_CLASS_SCOPE_TEXT)}",
|
||
f"- 大车 two-face 对比样本: {int(large_vehicle_summary.get('yaw_compare_count', 0) or 0)}",
|
||
f"- 大车 edge 更优占比: {format_percent(large_vehicle_summary.get('yaw_compare_edge_better_rate'))}",
|
||
f"- yaw>5deg 占比: {format_percent(overall['yaw_bad_rate'])}",
|
||
f"- horizontal>0.5m 占比: {format_percent(overall['x_bad_rate'])}",
|
||
f"- vertical(z)>0.5m 占比: {format_percent(overall['z_bad_rate'])}",
|
||
"",
|
||
"## 2D 阈值建议",
|
||
"",
|
||
f"- 本报告里所有依赖检测结果的统计统一使用 `conf > {format_float(to_float(payload.get('report_confidence_2d')), 3)}`。",
|
||
f"- 建议来源: ROI 的 F1-confidence 曲线最大 mean class F1,而不是默认 `{format_float(to_float(payload.get('configured_confidence_2d')), 3)}`。",
|
||
f"- 阈值建议曲线: `{confidence_curve_paths_2d.get('f1_curve', 'n/a')}`",
|
||
"",
|
||
"## 分类别结论",
|
||
"",
|
||
]
|
||
for item in per_class_insights:
|
||
lines.extend(
|
||
[
|
||
f"### {item['cls_name']} (id={item['cls_id']})",
|
||
"",
|
||
f"- 匹配样本: 3D={item['matched_3d']}, 位置有效={item['matched_pos']}",
|
||
f"- 全局误差: yaw={format_float(item['yaw_mae_deg'], 2)}deg, "
|
||
f"x={format_float(item['x_abs_mae_m'], 3)}m, z={format_float(item['z_abs_mae_m'], 3)}m",
|
||
f"- 超阈值占比: yaw>5deg={format_percent(item['yaw_bad_rate'])}, "
|
||
f"x>0.5m={format_percent(item['x_bad_rate'])}, z>0.5m={format_percent(item['z_bad_rate'])}",
|
||
f"- 水平误差最差横向 5m 区间: {item['horizontal_bins_text']}",
|
||
f"- 垂直误差最差 5m 深度区间: {item['vertical_bins_text']}",
|
||
f"- 偏航误差最差 10m 深度区间: {item['yaw_bins_text']}",
|
||
"",
|
||
]
|
||
)
|
||
lines.extend(
|
||
[
|
||
"## 明细文件",
|
||
"",
|
||
f"- 水平误差横向 5m 分桶: `{horizontal_csv}`",
|
||
f"- 垂直误差 5m 深度分桶: `{vertical_csv}`",
|
||
f"- 偏航误差 10m 深度分桶: `{yaw_depth_csv}`",
|
||
f"- 偏航误差 GT yaw 10deg 分桶: `{yaw_heading_csv}`",
|
||
f"- 偏航对比横向 5m 分桶: `{path.parent / 'yaw_compare_signed_lateral_5m.csv'}`",
|
||
f"- 偏航对比分 face-visibility 横向 5m 分桶: `{path.parent / 'yaw_compare_signed_lateral_5m_by_face_visibility.csv'}`",
|
||
f"- face/cut 选择准确率: `{path.parent / 'face_selection_accuracy.csv'}`",
|
||
f"- 遮挡二分类准确率: `{path.parent / 'occlusion_binary_accuracy.csv'}`",
|
||
f"- 分类别整体指标: `{path.parent / 'class_metrics.csv'}`",
|
||
f"- 2D F1 曲线: `{confidence_curve_paths_2d.get('f1_curve', 'n/a')}`",
|
||
f"- 近距离无遮挡 confusion matrix: `{payload.get('focused_confusion_matrix_plot_path_2d', 'n/a')}`",
|
||
f"- 坏例汇总: `{path.parent / 'bad_cases_yaw.csv'}`, `{path.parent / 'bad_cases_horizontal.csv'}`, `{path.parent / 'bad_cases_vertical.csv'}`",
|
||
f"- 可视化目录: `{path.parent / 'visuals'}`",
|
||
]
|
||
)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def sort_nullable_desc(rows: Iterable[tuple[str, dict[str, Any]]], key_name: str) -> list[tuple[str, dict[str, Any]]]:
|
||
def key_fn(item):
|
||
value = item[1].get(key_name)
|
||
if value is None:
|
||
return float("-inf")
|
||
return float(value)
|
||
|
||
return sorted(rows, key=key_fn, reverse=True)
|
||
|
||
|
||
def write_combined_summary_zh(path: Path, data_yaml: Path, split_path: Path, image_root: Path, summary_by_roi: dict[str, Any]) -> None:
|
||
lines = [
|
||
"# 双 ROI 验证集误差统计总结",
|
||
"",
|
||
"## 数据与口径",
|
||
"",
|
||
f"- 数据配置: `{data_yaml}`",
|
||
f"- 验证列表: `{split_path}`",
|
||
f"- 图像根目录: `{image_root}`",
|
||
"- 匹配规则: 同类别 + IoU>=0.5。",
|
||
"- 区间解释: 1m / 5m / 10m interval 均按 GT 前向距离 `z_gt` 分桶。",
|
||
"- 坏例导出规则: yaw 使用 `>5deg`,位置使用 `>0.5m`;图像导出采用分桶抽样,CSV 保留全部超阈值样本。",
|
||
"",
|
||
"## ROI 总览",
|
||
"",
|
||
]
|
||
for roi_name, payload in summary_by_roi.items():
|
||
overall = payload["overall"]
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
focused_confusion_filter = payload.get("focused_confusion_filter") or {}
|
||
lines.extend(
|
||
[
|
||
f"### {roi_name.upper()}",
|
||
"",
|
||
f"- matched_3d={overall['matched_3d']}, matched_pos={overall['matched_pos']}",
|
||
f"- 2D 建议阈值={format_float(to_float(threshold_advice_2d.get('recommended_confidence')), 3)}, "
|
||
f"precision={format_percent(overall.get('precision_2d'))}, recall={format_percent(overall.get('recall_2d'))}, F1={format_percent(overall.get('f1_2d'))}",
|
||
f"- 近距离无遮挡纵向范围: |z|<{format_float(to_float(focused_confusion_filter.get('max_abs_longitudinal_m')), 1)}m",
|
||
f"- yaw_mae={format_float(overall['yaw_mae_deg'], 3)}deg, yaw>5deg={format_percent(overall['yaw_bad_rate'])}",
|
||
f"- x_mae={format_float(overall['x_abs_mae_m'], 3)}m, x>0.5m={format_percent(overall['x_bad_rate'])}",
|
||
f"- z_mae={format_float(overall['z_abs_mae_m'], 3)}m, z>0.5m={format_percent(overall['z_bad_rate'])}",
|
||
f"- yaw 主要贡献类别: " + " / ".join(
|
||
f"{row['cls_name']}({format_float(row.get('yaw_mae_deg'), 1)}deg, 占比{format_percent(row.get('yaw_contribution_rate'))})"
|
||
for row in payload["top_yaw_contributors"][:4]
|
||
if row.get("matched_3d")
|
||
),
|
||
"",
|
||
]
|
||
)
|
||
lines.extend(["## 分 ROI 细结论", ""])
|
||
for roi_name, payload in summary_by_roi.items():
|
||
lines.extend(
|
||
[
|
||
f"### {roi_name.upper()}",
|
||
"",
|
||
f"- 中文详版: `{path.parent / roi_name / 'summary_zh.md'}`",
|
||
f"- 英文简版: `{path.parent / roi_name / 'summary.md'}`",
|
||
]
|
||
)
|
||
for item in payload["per_class_interval_insights"]:
|
||
lines.extend(
|
||
[
|
||
f"- {item['cls_name']}: yaw={format_float(item['yaw_mae_deg'], 2)}deg, "
|
||
f"x={format_float(item['x_abs_mae_m'], 3)}m, z={format_float(item['z_abs_mae_m'], 3)}m; "
|
||
f"最差水平区间={item['horizontal_bins_text']}; "
|
||
f"最差垂直区间={item['vertical_bins_text']}; "
|
||
f"最差偏航区间={item['yaw_bins_text']}",
|
||
]
|
||
)
|
||
lines.append("")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def interval_row_key(row: dict[str, Any]) -> tuple[int, str, float]:
|
||
return (
|
||
int(row["cls_id"]),
|
||
str(row["cls_name"]),
|
||
float(row.get("interval_start", row.get("depth_bin_start_m", 0.0))),
|
||
)
|
||
|
||
|
||
def compute_interval_top_keys(rows: list[dict[str, Any]], metric_key: str, topn: int = 3, min_count: int = 30) -> set[tuple[int, str, float]]:
|
||
top_keys: set[tuple[int, str, float]] = set()
|
||
for _, class_rows in group_rows_by_class(rows).items():
|
||
for row in top_interval_rows(class_rows, metric_key=metric_key, topn=topn, min_count=min_count):
|
||
top_keys.add(interval_row_key(row))
|
||
return top_keys
|
||
|
||
|
||
def build_yaw_compare_diagnostics(
|
||
overall_summary: dict[str, Any],
|
||
breakdown_summaries: dict[str, dict[str, dict[str, Any]]],
|
||
) -> dict[str, Any]:
|
||
focus_bucket = "two-face"
|
||
face_visibility_summary = breakdown_summaries.get("face_visibility", {})
|
||
face_rows = []
|
||
for bucket_name in FACE_VISIBILITY_BUCKET_ORDER:
|
||
row = dict(face_visibility_summary.get(bucket_name) or summarize_metric_bucket(make_metric_bucket()))
|
||
row["key"] = bucket_name
|
||
face_rows.append(row)
|
||
|
||
face_row_map = {str(row["key"]): row for row in face_rows}
|
||
dominant_face = next((row for row in face_rows if int(row.get("yaw_compare_count") or 0) > 0), None)
|
||
focus_face = face_row_map.get(focus_bucket, {})
|
||
compare_count = int(focus_face.get("yaw_compare_count") or 0)
|
||
matched_3d = int(overall_summary.get("matched_3d") or 0)
|
||
subset_rate = rate_or_none(compare_count, matched_3d)
|
||
|
||
explanation_parts = [
|
||
"All yaw errors in this report use pi-periodic orientation error, so 180-degree front-rear flips are treated as near-zero orientation error.",
|
||
"Both direct-regression and edge-based yaw metrics in this report are compared against gt_yaw.",
|
||
"The face-visibility diagnostic splits paired samples into front_rear_only, side only, and two-face buckets.",
|
||
"For front_rear_only and side only buckets, the diagnostic uses the best available single-face edge yaw so those rows can still compare direct versus edge-based yaw.",
|
||
f"Signed lateral yaw-compare bins use {format_float(HORIZONTAL_LATERAL_BIN_M, 1)}m bins inside the configured lateral range.",
|
||
(
|
||
f"In this ROI, the paired subset is {compare_count}/{matched_3d} "
|
||
f"({format_percent(subset_rate)}) of matched_3d."
|
||
),
|
||
]
|
||
if focus_face:
|
||
explanation_parts.append(
|
||
f"The focused two-face bucket has direct={format_float(focus_face.get('direct_regression_yaw_mae_deg'), 2)} deg and "
|
||
f"edge={format_float(focus_face.get('edge_based_yaw_mae_deg'), 2)} deg over {compare_count} pairs."
|
||
)
|
||
if int(focus_face.get("length_compare_count") or 0) > 0:
|
||
explanation_parts.append(
|
||
f"Its side-edge length MAE is {format_float(focus_face.get('side_edge_length_mae_m'), 3)} m versus "
|
||
f"{format_float(focus_face.get('direct_regression_length_mae_m'), 3)} m from direct regression."
|
||
)
|
||
if dominant_face is not None:
|
||
explanation_parts.append(
|
||
f"The largest face-visibility bucket is {dominant_face['key']} with {dominant_face['yaw_compare_count']} pairs "
|
||
f"(direct={format_float(dominant_face.get('direct_regression_yaw_mae_deg'), 2)} deg, "
|
||
f"edge={format_float(dominant_face.get('edge_based_yaw_mae_deg'), 2)} deg)."
|
||
)
|
||
|
||
return {
|
||
"focus_bucket": focus_bucket,
|
||
"focus_face_row": focus_face,
|
||
"compare_count": compare_count,
|
||
"matched_3d": matched_3d,
|
||
"paired_subset_rate": subset_rate,
|
||
"overall_yaw_mae_deg": overall_summary.get("yaw_mae_deg"),
|
||
"face_visibility_rows": face_rows,
|
||
"explanation": " ".join(part for part in explanation_parts if part),
|
||
}
|
||
|
||
|
||
def html_escape(value: Any) -> str:
|
||
return html_lib.escape("" if value is None else str(value), quote=True)
|
||
|
||
|
||
def manual_summary_key(value: Any, fallback: str = "manual-summary") -> str:
|
||
slug = "".join(char.lower() if char.isalnum() else "-" for char in ("" if value is None else str(value)).strip())
|
||
while "--" in slug:
|
||
slug = slug.replace("--", "-")
|
||
slug = slug.strip("-")
|
||
return slug or fallback
|
||
|
||
|
||
def render_manual_summary_slot(section_title: Optional[str] = None) -> str:
|
||
section_label = str(section_title).strip() if section_title else "Manual Summary"
|
||
section_key = manual_summary_key(section_label)
|
||
return (
|
||
f"<div class='manual-summary' title='Manual summary placeholder' "
|
||
f"data-manual-summary-slot='true' data-manual-summary-key='{html_escape(section_key)}' "
|
||
f"data-manual-summary-label='{html_escape(section_label)}'>"
|
||
f"<!-- MANUAL_SUMMARY_SLOT key={section_key} -->"
|
||
"</div>"
|
||
)
|
||
|
||
|
||
def format_bool(value: Any) -> str:
|
||
return "yes" if bool(value) else "no"
|
||
|
||
|
||
def relative_href(report_path: Path, target_path: str | Path) -> str:
|
||
return html_escape(os.path.relpath(str(Path(target_path)), start=str(report_path.parent)))
|
||
|
||
|
||
def load_manifest_records(path: str | Path) -> list[dict[str, Any]]:
|
||
manifest_path = Path(path)
|
||
if not manifest_path.is_file():
|
||
return []
|
||
with manifest_path.open("r", encoding="utf-8") as file:
|
||
payload = json.load(file)
|
||
return payload if isinstance(payload, list) else []
|
||
|
||
|
||
def render_html_table(title: str, headers: list[str], rows: list[list[str]], table_class: str = "report-table") -> str:
|
||
if not rows:
|
||
return (
|
||
f"<section class='table-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>No data.</p></section>"
|
||
)
|
||
head_html = "".join(f"<th>{html_escape(header)}</th>" for header in headers)
|
||
body_html = []
|
||
for row in rows:
|
||
body_html.append("<tr>" + "".join(f"<td>{cell}</td>" for cell in row) + "</tr>")
|
||
return (
|
||
f"<section class='table-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}"
|
||
f"<div class='table-wrap'><table class='{html_escape(table_class)}'><thead><tr>{head_html}</tr></thead>"
|
||
f"<tbody>{''.join(body_html)}</tbody></table></div></section>"
|
||
)
|
||
|
||
|
||
def render_metric_value_table(title: str, rows: list[tuple[str, str]]) -> str:
|
||
return render_html_table(title, ["Metric", "Value"], [[html_escape(metric), value] for metric, value in rows], table_class="metric-table")
|
||
|
||
|
||
def _pie_palette() -> list[str]:
|
||
return ["#0e7490", "#1f6f50", "#c26d1a", "#8b5e3c", "#8e3b46", "#5b6cfa", "#0f766e", "#7c3aed", "#64748b"]
|
||
|
||
|
||
def build_pie_segments(
|
||
rows: list[dict[str, Any]],
|
||
label_key: str,
|
||
value_key: str,
|
||
max_segments: int = 7,
|
||
) -> tuple[list[dict[str, Any]], int]:
|
||
nonzero = [
|
||
{"label": str(row.get(label_key, "")), "count": max(0, int(row.get(value_key, 0) or 0))}
|
||
for row in rows
|
||
if max(0, int(row.get(value_key, 0) or 0)) > 0
|
||
]
|
||
total = int(sum(int(item["count"]) for item in nonzero))
|
||
if not nonzero or total <= 0:
|
||
return [], total
|
||
|
||
ranked = sorted(nonzero, key=lambda item: (int(item["count"]), item["label"]), reverse=True)
|
||
if len(ranked) > max_segments:
|
||
kept = ranked[: max_segments - 1]
|
||
other_count = int(sum(int(item["count"]) for item in ranked[max_segments - 1 :]))
|
||
if other_count > 0:
|
||
kept.append({"label": "Other", "count": other_count})
|
||
ranked = kept
|
||
return ranked, total
|
||
|
||
|
||
def polar_to_cartesian(cx: float, cy: float, radius: float, angle_deg: float) -> tuple[float, float]:
|
||
angle_rad = math.radians(float(angle_deg))
|
||
return float(cx + radius * math.cos(angle_rad)), float(cy + radius * math.sin(angle_rad))
|
||
|
||
|
||
def pie_slice_path(cx: float, cy: float, radius: float, start_angle_deg: float, end_angle_deg: float) -> str:
|
||
start_x, start_y = polar_to_cartesian(cx, cy, radius, start_angle_deg)
|
||
end_x, end_y = polar_to_cartesian(cx, cy, radius, end_angle_deg)
|
||
large_arc = 1 if (end_angle_deg - start_angle_deg) > 180.0 else 0
|
||
return (
|
||
f"M {cx:.1f} {cy:.1f} "
|
||
f"L {start_x:.1f} {start_y:.1f} "
|
||
f"A {radius:.1f} {radius:.1f} 0 {large_arc} 1 {end_x:.1f} {end_y:.1f} Z"
|
||
)
|
||
|
||
|
||
def render_count_bar_svg(
|
||
rows: list[dict[str, Any]],
|
||
label_key: str,
|
||
value_key: str,
|
||
x_axis_title: str,
|
||
y_axis_title: str,
|
||
aria_label: str,
|
||
fill_color: str = "#0e7490",
|
||
) -> str:
|
||
if not rows:
|
||
return "<p class='muted'>No distribution data.</p>"
|
||
|
||
ordered = list(rows)
|
||
pie_segments, total_count = build_pie_segments(ordered, label_key, value_key)
|
||
max_value = max((max(0, int(row.get(value_key, 0) or 0)) for row in ordered), default=0)
|
||
max_value = max(1, max_value)
|
||
|
||
left = 74
|
||
right = 16
|
||
top = 18
|
||
bottom = 96
|
||
plot_height = 190
|
||
bar_pitch = 30 if len(ordered) <= 24 else 26
|
||
plot_width = max(760, len(ordered) * bar_pitch)
|
||
chart_right = left + plot_width
|
||
pie_panel_x = chart_right + 26
|
||
pie_panel_width = 320
|
||
bar_width = max(10.0, min(22.0, bar_pitch * 0.72))
|
||
major_tick_count = 4
|
||
pie_radius = 74.0
|
||
pie_center_x = pie_panel_x + 104.0
|
||
pie_center_y = top + 92.0
|
||
legend_y = pie_center_y + pie_radius + 18.0
|
||
legend_step = 18.0
|
||
height = int(max(top + plot_height + bottom, legend_y + max(1, len(pie_segments)) * legend_step + 20.0))
|
||
width = int(chart_right + pie_panel_width + right)
|
||
palette = _pie_palette()
|
||
|
||
parts = [
|
||
f"<svg class='histogram-svg' viewBox='0 0 {width} {height}' role='img' aria-label='{html_escape(aria_label)}'>",
|
||
f"<line x1='{left}' y1='{top + plot_height}' x2='{chart_right}' y2='{top + plot_height}' class='axis-line' />",
|
||
f"<line x1='{left}' y1='{top}' x2='{left}' y2='{top + plot_height}' class='axis-line' />",
|
||
f"<line x1='{pie_panel_x - 10}' y1='{top}' x2='{pie_panel_x - 10}' y2='{height - 24}' class='grid-line' />",
|
||
]
|
||
|
||
for tick_idx in range(major_tick_count + 1):
|
||
tick_value = max_value * tick_idx / float(major_tick_count)
|
||
y = top + plot_height - (tick_value / max_value) * plot_height
|
||
parts.append(f"<line x1='{left}' y1='{y:.1f}' x2='{chart_right}' y2='{y:.1f}' class='grid-line' />")
|
||
parts.append(
|
||
f"<text x='{left - 8}' y='{y + 4:.1f}' text-anchor='end' class='axis-text'>{html_escape(str(int(round(tick_value))))}</text>"
|
||
)
|
||
|
||
label_stride = max(1, math.ceil(len(ordered) / 24))
|
||
for idx, row in enumerate(ordered):
|
||
count = max(0, int(row.get(value_key, 0) or 0))
|
||
label = str(row.get(label_key, ""))
|
||
bar_height = (count / max_value) * plot_height if max_value > 0 else 0.0
|
||
x = left + idx * bar_pitch + (bar_pitch - bar_width) / 2.0
|
||
y = top + plot_height - bar_height
|
||
title = f"{label}\ncount={count}"
|
||
parts.append(
|
||
f"<rect x='{x:.1f}' y='{y:.1f}' width='{bar_width:.1f}' height='{bar_height:.1f}' rx='4' ry='4' fill='{html_escape(fill_color)}' opacity='0.88'>"
|
||
f"<title>{html_escape(title)}</title></rect>"
|
||
)
|
||
if count > 0:
|
||
text_y = max(top + 11.0, y - 6.0)
|
||
parts.append(
|
||
f"<text x='{x + bar_width / 2.0:.1f}' y='{text_y:.1f}' text-anchor='middle' class='count-default'>{html_escape(str(count))}</text>"
|
||
)
|
||
if len(ordered) <= 12 or idx % label_stride == 0 or count == max_value:
|
||
label_y = top + plot_height + 14
|
||
parts.append(
|
||
f"<text x='{x + bar_width / 2.0:.1f}' y='{label_y:.1f}' transform='rotate(55 {x + bar_width / 2.0:.1f} {label_y:.1f})' "
|
||
f"text-anchor='start' class='bin-label'>{html_escape(label)}</text>"
|
||
)
|
||
|
||
if pie_segments and total_count > 0:
|
||
angle_start = -90.0
|
||
for index, segment in enumerate(pie_segments):
|
||
ratio = float(segment["count"]) / float(total_count)
|
||
angle_end = angle_start + ratio * 360.0
|
||
color = "#94a3b8" if segment["label"] == "Other" else palette[index % len(palette)]
|
||
if math.isclose(ratio, 1.0, rel_tol=0.0, abs_tol=1e-9):
|
||
parts.append(
|
||
f"<circle cx='{pie_center_x:.1f}' cy='{pie_center_y:.1f}' r='{pie_radius:.1f}' fill='{html_escape(color)}'>"
|
||
f"<title>{html_escape(segment['label'])}: {segment['count']} ({format_percent(ratio)})</title></circle>"
|
||
)
|
||
else:
|
||
parts.append(
|
||
f"<path d='{pie_slice_path(pie_center_x, pie_center_y, pie_radius, angle_start, angle_end)}' fill='{html_escape(color)}'>"
|
||
f"<title>{html_escape(segment['label'])}: {segment['count']} ({format_percent(ratio)})</title></path>"
|
||
)
|
||
angle_start = angle_end
|
||
|
||
parts.append(f"<circle cx='{pie_center_x:.1f}' cy='{pie_center_y:.1f}' r='34' fill='{html_escape('#fffdf7')}' />")
|
||
parts.append(
|
||
f"<text x='{pie_center_x:.1f}' y='{pie_center_y - 6:.1f}' text-anchor='middle' class='axis-title'>Total</text>"
|
||
)
|
||
parts.append(
|
||
f"<text x='{pie_center_x:.1f}' y='{pie_center_y + 14:.1f}' text-anchor='middle' class='axis-text'>{html_escape(str(total_count))}</text>"
|
||
)
|
||
for index, segment in enumerate(pie_segments):
|
||
color = "#94a3b8" if segment["label"] == "Other" else palette[index % len(palette)]
|
||
ratio = float(segment["count"]) / float(total_count)
|
||
legend_row_y = legend_y + index * legend_step
|
||
parts.append(
|
||
f"<rect x='{pie_panel_x:.1f}' y='{legend_row_y - 9:.1f}' width='12' height='12' rx='2' ry='2' fill='{html_escape(color)}' />"
|
||
)
|
||
parts.append(
|
||
f"<text x='{pie_panel_x + 18:.1f}' y='{legend_row_y:.1f}' class='axis-text'>"
|
||
f"{html_escape(segment['label'])}: {html_escape(str(segment['count']))} ({html_escape(format_percent(ratio))})"
|
||
"</text>"
|
||
)
|
||
else:
|
||
parts.append(
|
||
f"<text x='{pie_panel_x + pie_panel_width / 2.0:.1f}' y='{pie_center_y:.1f}' text-anchor='middle' class='axis-text'>No nonzero slices</text>"
|
||
)
|
||
|
||
parts.append(
|
||
f"<text x='{left + plot_width / 2.0:.1f}' y='{height - 18}' text-anchor='middle' class='axis-title'>{html_escape(x_axis_title)}</text>"
|
||
)
|
||
parts.append(
|
||
f"<text x='18' y='{top + plot_height / 2.0:.1f}' transform='rotate(-90 18 {top + plot_height / 2.0:.1f})' "
|
||
f"text-anchor='middle' class='axis-title'>{html_escape(y_axis_title)}</text>"
|
||
)
|
||
parts.append(
|
||
f"<text x='{pie_panel_x + pie_panel_width / 2.0:.1f}' y='{top + 6:.1f}' text-anchor='middle' class='axis-title'>Percentage distribution</text>"
|
||
)
|
||
parts.append("</svg>")
|
||
return "".join(parts)
|
||
|
||
|
||
def render_distribution_cards_section(
|
||
title: str,
|
||
cards: list[dict[str, Any]],
|
||
label_key: str,
|
||
value_key: str,
|
||
x_axis_title: str,
|
||
y_axis_title: str,
|
||
fill_color: str,
|
||
note: str = "",
|
||
) -> str:
|
||
if not cards:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>No distribution data.</p></section>"
|
||
)
|
||
|
||
note_html = f"<p class='muted'>{html_escape(note)}</p>" if note else ""
|
||
card_html = []
|
||
for card in cards:
|
||
subtitle = str(card.get("subtitle", "") or "")
|
||
subtitle_html = f"<span>{html_escape(subtitle)}</span>" if subtitle else ""
|
||
card_html.append(
|
||
"<article class='hist-card'>"
|
||
f"<h4>{html_escape(str(card.get('title', 'Untitled')))}{subtitle_html}</h4>"
|
||
f"{render_count_bar_svg(card.get('rows', []), label_key, value_key, x_axis_title, y_axis_title, title, fill_color=fill_color)}"
|
||
"</article>"
|
||
)
|
||
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}{note_html}<div class='hist-grid'>{''.join(card_html)}</div></section>"
|
||
)
|
||
|
||
|
||
def render_collapsible_distribution_groups(
|
||
title: str,
|
||
groups: list[dict[str, Any]],
|
||
label_key: str,
|
||
value_key: str,
|
||
x_axis_title: str,
|
||
y_axis_title: str,
|
||
fill_color: str,
|
||
note: str = "",
|
||
) -> str:
|
||
if not groups:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>No grouped distributions.</p></section>"
|
||
)
|
||
|
||
note_html = f"<p class='muted'>{html_escape(note)}</p>" if note else ""
|
||
group_html = []
|
||
for group in groups:
|
||
rows = group.get("rows", [])
|
||
if not rows:
|
||
continue
|
||
subtitle = str(group.get("subtitle", "") or "")
|
||
subtitle_html = f"<span>{html_escape(subtitle)}</span>" if subtitle else ""
|
||
group_html.append(
|
||
f"<details class='badcase-group'><summary>{html_escape(str(group.get('summary', 'group')))}</summary>"
|
||
"<div class='hist-grid'>"
|
||
"<article class='hist-card'>"
|
||
f"<h4>{html_escape(str(group.get('title', group.get('summary', 'group'))))}{subtitle_html}</h4>"
|
||
f"{render_count_bar_svg(rows, label_key, value_key, x_axis_title, y_axis_title, str(group.get('title', title)), fill_color=fill_color)}"
|
||
"</article></div>"
|
||
"</details>"
|
||
)
|
||
|
||
if not group_html:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>No grouped distributions.</p></section>"
|
||
)
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}{note_html}{''.join(group_html)}</section>"
|
||
)
|
||
|
||
|
||
def render_data_portrait_section(portrait_payload: dict[str, Any]) -> str:
|
||
split_name = str(portrait_payload.get("split", "train"))
|
||
split_label = split_name.upper()
|
||
summary = portrait_payload.get("summary") or {}
|
||
vehicle_rows = portrait_payload.get("vehicle_rows", [])
|
||
class_total_rows = portrait_payload.get("class_total_rows", [])
|
||
class_vehicle_groups = portrait_payload.get("class_vehicle_groups", [])
|
||
vehicle_cards = [
|
||
{
|
||
"title": "All Vehicles",
|
||
"subtitle": (
|
||
f"total_frames={int(summary.get('num_entries', 0) or 0)} "
|
||
f"vehicles={int(summary.get('vehicles', 0) or 0)}"
|
||
),
|
||
"rows": [{"vehicle_alias": str(row.get("vehicle_alias", "")), "frame_count": int(row.get("frame_count", 0) or 0)} for row in vehicle_rows],
|
||
}
|
||
]
|
||
class_total_cards = [
|
||
{
|
||
"title": "All Vehicles",
|
||
"subtitle": (
|
||
f"class_frame_hits={sum(int(row.get('frame_count', 0) or 0) for row in class_total_rows)} "
|
||
f"objects={sum(int(row.get('object_count', 0) or 0) for row in class_total_rows)}"
|
||
),
|
||
"rows": [
|
||
{"cls_name": str(row.get("cls_name", "")), "frame_count": int(row.get("frame_count", 0) or 0)}
|
||
for row in class_total_rows
|
||
if int(row.get("frame_count", 0) or 0) > 0
|
||
],
|
||
}
|
||
]
|
||
class_group_charts = [
|
||
{
|
||
"summary": str(group.get("summary", "group")),
|
||
"title": str(group.get("vehicle_alias", group.get("summary", "group"))),
|
||
"subtitle": (
|
||
f"class_frame_hits={sum(int(row.get('frame_count', 0) or 0) for row in group.get('rows', []))} "
|
||
f"objects={sum(int(row.get('object_count', 0) or 0) for row in group.get('rows', []))}"
|
||
),
|
||
"rows": [
|
||
{"cls_name": str(row.get("cls_name", "")), "frame_count": int(row.get("frame_count", 0) or 0)}
|
||
for row in group.get("rows", [])
|
||
if int(row.get("frame_count", 0) or 0) > 0
|
||
],
|
||
}
|
||
for group in class_vehicle_groups
|
||
]
|
||
portrait_overview_html = (
|
||
f"<section class='gallery-section'><h3>{html_escape(split_label)} Portrait Overview</h3>"
|
||
f"{render_manual_summary_slot(f'{split_label} Portrait Overview')}"
|
||
"<p class='muted'>"
|
||
f"Split <code>{html_escape(split_name)}</code> from <code>{html_escape(str(portrait_payload.get('split_path', 'n/a')))}</code> "
|
||
f"with image root <code>{html_escape(str(portrait_payload.get('image_root', 'n/a')))}</code>. "
|
||
f"Entries={int(summary.get('num_entries', 0) or 0)}, vehicles={int(summary.get('vehicles', 0) or 0)}, "
|
||
f"frames_with_valid_3d={int(summary.get('frames_with_valid_3d', 0) or 0)}, mapped_objects={int(summary.get('mapped_objects', 0) or 0)}, "
|
||
f"heading_objects={int(summary.get('heading_objects', 0) or 0)}, lateral_objects={int(summary.get('lateral_objects', 0) or 0)}, "
|
||
f"longitudinal_objects={int(summary.get('longitudinal_objects', 0) or 0)}."
|
||
"</p>"
|
||
"<p class='muted'>"
|
||
f"Portrait summary JSON: <code>{html_escape(str(portrait_payload.get('summary_path', 'n/a')))}</code>. "
|
||
"Every portrait chart below shows raw counts in the histogram and percentage distribution in the pie chart."
|
||
"</p></section>"
|
||
)
|
||
|
||
return (
|
||
"<section class='tab-panel' id='portrait'>"
|
||
f"<h2>{html_escape(split_label)} Portrait</h2>"
|
||
"<p class='muted'>"
|
||
"Frame-level statistics decode vehicle alias and capture time from GT filenames. "
|
||
"Object-level statistics reuse the dataset class map and GT 3D decoding logic from this report pipeline."
|
||
"</p>"
|
||
f"{portrait_overview_html}"
|
||
f"{render_distribution_cards_section('Frames Per Vehicle', vehicle_cards, 'vehicle_alias', 'frame_count', 'Vehicle alias', 'Frames', '#0e7490', note='Each vehicle bar shows frame count, while the pie chart summarizes the frame-share percentage by vehicle alias.')}"
|
||
f"{render_distribution_cards_section('Frames Per Day', portrait_payload.get('daily_cards', []), 'day', 'frame_count', 'Capture day', 'Frames', '#0e7490', note='The first card is the total distribution across all vehicles; following cards keep the same day axis per vehicle.')}"
|
||
f"{render_distribution_cards_section('Frames Per Hour', portrait_payload.get('hourly_cards', []), 'hour', 'frame_count', 'Hour of day', 'Frames', '#1f6f50', note='Hours are aggregated by local capture hour decoded from the GT filename timestamp.')}"
|
||
f"{render_distribution_cards_section('Class Frame Distribution Across All Vehicles', class_total_cards, 'cls_name', 'frame_count', 'Object class', 'Frames', '#8e3b46', note='This distribution uses frame_count, meaning each class counts how many frames contain at least one instance of that class.')}"
|
||
f"{render_collapsible_distribution_groups('Class Frame Distribution By Vehicle', class_group_charts, 'cls_name', 'frame_count', 'Object class', 'Frames', '#8e3b46', note='Each collapsed vehicle card uses frame_count for the histogram and pie chart; subtitles also show the corresponding object totals.')}"
|
||
f"{render_distribution_cards_section('Heading Distribution (10deg bins)', portrait_payload.get('heading_cards', []), 'heading_bin_label', 'count', 'Heading bin', 'Objects', '#c26d1a', note='Heading uses GT yaw wrapped into [-180, 180) and counted with 10-degree bins. The first card is the total without vehicle alias.')}"
|
||
f"{render_distribution_cards_section('Lateral Distance By Class (|x|, 5m bins)', portrait_payload.get('lateral_cards', []), 'lateral_bin_label', 'count', 'Absolute lateral distance bin', 'Objects', '#0f766e', note='Each card aggregates all vehicles for one class; lateral distance uses the absolute GT x position in meters.')}"
|
||
f"{render_distribution_cards_section('Longitudinal Distance By Class (z, 10m bins)', portrait_payload.get('longitudinal_cards', []), 'longitudinal_bin_label', 'count', 'Longitudinal distance bin', 'Objects', '#8b5e3c', note='Each card aggregates all vehicles for one class; longitudinal distance uses GT depth z in meters.')}"
|
||
"</section>"
|
||
)
|
||
|
||
|
||
def highlight_if_needed(value: str, highlight: bool) -> str:
|
||
if not highlight:
|
||
return value
|
||
return f"<span class='emphasis-red'>{value}</span>"
|
||
|
||
|
||
def semantic_emphasis(value: str, css_class: Optional[str]) -> str:
|
||
if not css_class:
|
||
return value
|
||
return f"<span class='{html_escape(css_class)}'>{value}</span>"
|
||
|
||
|
||
def format_signed_float(value: Optional[float], digits: int = 2) -> str:
|
||
if value is None or not math.isfinite(value):
|
||
return "n/a"
|
||
return f"{value:+.{digits}f}"
|
||
|
||
|
||
def render_edge_gain_deg(value: Optional[float], digits: int = 2) -> str:
|
||
text = html_escape(format_signed_float(value, digits))
|
||
if value is None or not math.isfinite(value):
|
||
return text
|
||
if value < 0:
|
||
return semantic_emphasis(text, "emphasis-green")
|
||
if value > 0:
|
||
return semantic_emphasis(text, "emphasis-red")
|
||
return text
|
||
|
||
|
||
def render_rate_vs_half(value: Optional[float], digits: int = 1) -> str:
|
||
text = html_escape(format_percent(value, digits))
|
||
if value is None or not math.isfinite(value):
|
||
return text
|
||
if value > 0.5:
|
||
return semantic_emphasis(text, "emphasis-green")
|
||
if value < 0.5:
|
||
return semantic_emphasis(text, "emphasis-red")
|
||
return text
|
||
|
||
|
||
def top_bad_class_ids(
|
||
rows: list[dict[str, Any]],
|
||
metric_key: str,
|
||
*,
|
||
higher_is_worse: bool,
|
||
support_key: Optional[str] = None,
|
||
topn: int = 3,
|
||
) -> set[int]:
|
||
ranked: list[tuple[float, int, int]] = []
|
||
for row in rows:
|
||
cls_id = int(row.get("cls_id", -1))
|
||
value = row.get(metric_key)
|
||
if value is None:
|
||
continue
|
||
value_f = to_float(value)
|
||
if value_f is None:
|
||
continue
|
||
support = int(row.get(support_key) or 0) if support_key else 0
|
||
if support_key and support <= 0:
|
||
continue
|
||
ranked.append((float(value_f), support, cls_id))
|
||
|
||
if higher_is_worse:
|
||
ranked.sort(key=lambda item: (item[0], item[1], -item[2]), reverse=True)
|
||
else:
|
||
ranked.sort(key=lambda item: (item[0], -item[1], item[2]))
|
||
return {cls_id for _, _, cls_id in ranked[:topn]}
|
||
|
||
|
||
def render_interval_trend_svg(
|
||
rows: list[dict[str, Any]],
|
||
metric_key: str,
|
||
top_keys: set[tuple[int, str, float]],
|
||
unit_suffix: str,
|
||
x_axis_title: str = "Depth bin",
|
||
relative_metric_key: Optional[str] = None,
|
||
relative_y_axis_title: str = "Relative error (%)",
|
||
relative_reference_label: str = "gt_bin",
|
||
relative_reference_unit: str = "m",
|
||
) -> str:
|
||
valid_rows = [row for row in rows if row.get(metric_key) is not None]
|
||
if not valid_rows:
|
||
return "<p class='muted'>No interval data.</p>"
|
||
|
||
ordered = sorted(valid_rows, key=lambda row: float(row.get("interval_start", row.get("depth_bin_start_m", 0.0))))
|
||
max_value = max(float(row[metric_key]) for row in ordered)
|
||
if not math.isfinite(max_value) or max_value <= 0:
|
||
max_value = 1.0
|
||
|
||
show_relative = bool(relative_metric_key) and any(to_float(row.get(relative_metric_key)) is not None for row in ordered)
|
||
max_relative_value = 1.0
|
||
if show_relative:
|
||
max_relative_value = max(float(to_float(row.get(relative_metric_key)) or 0.0) for row in ordered)
|
||
if not math.isfinite(max_relative_value) or max_relative_value <= 0:
|
||
max_relative_value = 1.0
|
||
|
||
left = 74
|
||
right = 24
|
||
top = 18
|
||
plot_height = 166 if show_relative else 190
|
||
relative_gap = 34 if show_relative else 0
|
||
relative_height = 104 if show_relative else 0
|
||
bottom = 98 if show_relative else 88
|
||
point_pitch = 26
|
||
width = max(860, left + right + len(ordered) * point_pitch)
|
||
height = top + plot_height + relative_gap + relative_height + bottom
|
||
plot_width = width - left - right
|
||
point_radius = 3.5
|
||
major_tick_count = 4
|
||
minor_subdivisions = 4
|
||
relative_tick_count = 3
|
||
abs_bottom = top + plot_height
|
||
relative_top = abs_bottom + relative_gap
|
||
relative_bottom = relative_top + relative_height
|
||
x_label_y = (relative_bottom if show_relative else abs_bottom) + 12
|
||
bar_width = min(18.0, point_pitch * 0.72)
|
||
|
||
def scaled_y(value: float, max_plot_value: float, panel_top: float, panel_height: float) -> float:
|
||
return panel_top + panel_height - (0.0 if max_plot_value <= 0 else (value / max_plot_value) * panel_height)
|
||
|
||
parts = [
|
||
f"<svg class='histogram-svg' viewBox='0 0 {width} {height}' role='img' aria-label='interval trend plot'>",
|
||
f"<line x1='{left}' y1='{abs_bottom}' x2='{left + plot_width}' y2='{abs_bottom}' class='axis-line' />",
|
||
f"<line x1='{left}' y1='{top}' x2='{left}' y2='{abs_bottom}' class='axis-line' />",
|
||
]
|
||
if show_relative:
|
||
parts.extend(
|
||
[
|
||
f"<line x1='{left}' y1='{relative_bottom}' x2='{left + plot_width}' y2='{relative_bottom}' class='axis-line' />",
|
||
f"<line x1='{left}' y1='{relative_top}' x2='{left}' y2='{relative_bottom}' class='axis-line' />",
|
||
]
|
||
)
|
||
|
||
total_minor_steps = major_tick_count * minor_subdivisions
|
||
for minor_idx in range(1, total_minor_steps):
|
||
if minor_idx % minor_subdivisions == 0:
|
||
continue
|
||
tick_value = max_value * minor_idx / float(total_minor_steps)
|
||
y = scaled_y(tick_value, max_value, top, plot_height)
|
||
parts.append(f"<line x1='{left}' y1='{y:.1f}' x2='{left + plot_width}' y2='{y:.1f}' class='grid-line-minor' />")
|
||
parts.append(
|
||
f"<text x='{left - 8}' y='{y + 3:.1f}' text-anchor='end' class='axis-text-minor'>{html_escape(format_float(tick_value, 2))}</text>"
|
||
)
|
||
|
||
for tick_idx in range(major_tick_count + 1):
|
||
tick_value = max_value * tick_idx / float(major_tick_count)
|
||
y = scaled_y(tick_value, max_value, top, plot_height)
|
||
parts.append(f"<line x1='{left}' y1='{y:.1f}' x2='{left + plot_width}' y2='{y:.1f}' class='grid-line' />")
|
||
parts.append(
|
||
f"<text x='{left - 8}' y='{y + 4:.1f}' text-anchor='end' class='axis-text'>{html_escape(format_float(tick_value, 2))}</text>"
|
||
)
|
||
|
||
if show_relative:
|
||
for tick_idx in range(relative_tick_count + 1):
|
||
tick_value = max_relative_value * tick_idx / float(relative_tick_count)
|
||
y = scaled_y(tick_value, max_relative_value, relative_top, relative_height)
|
||
parts.append(f"<line x1='{left}' y1='{y:.1f}' x2='{left + plot_width}' y2='{y:.1f}' class='grid-line-minor' />")
|
||
parts.append(
|
||
f"<text x='{left - 8}' y='{y + 4:.1f}' text-anchor='end' class='axis-text-minor'>{html_escape(format_float(tick_value, 1))}%</text>"
|
||
)
|
||
|
||
line_points = []
|
||
relative_line_points = []
|
||
relative_bar_parts = []
|
||
abs_point_parts = []
|
||
relative_point_parts = []
|
||
label_parts = []
|
||
label_stride = max(1, math.ceil(len(ordered) / 22))
|
||
for idx, row in enumerate(ordered):
|
||
value = float(row[metric_key])
|
||
y = scaled_y(value, max_value, top, plot_height)
|
||
x = left + idx * point_pitch + point_pitch / 2.0
|
||
is_top = interval_row_key(row) in top_keys
|
||
line_points.append(f"{x:.1f},{y:.1f}")
|
||
tooltip_lines = [
|
||
f"{row['cls_name']} {row.get('interval_label', row.get('depth_bin_label', 'n/a'))}\n"
|
||
f"count={row['count']}\n"
|
||
f"mean={format_float(row.get(metric_key), 3)}{unit_suffix}\n"
|
||
f"p90={format_float(row.get(metric_key.replace('mean_', 'p90_')), 3)}{unit_suffix}\n"
|
||
f"max={format_float(row.get(metric_key.replace('mean_', 'max_')), 3)}{unit_suffix}"
|
||
]
|
||
if show_relative and relative_metric_key:
|
||
tooltip_lines.append(
|
||
f"\n{relative_reference_label}={format_float(to_float(row.get('relative_reference_value')), 2)}{relative_reference_unit}\n"
|
||
f"relative_mean={format_float(row.get(relative_metric_key), 2)}%\n"
|
||
f"relative_p90={format_float(row.get(relative_metric_key.replace('mean_', 'p90_')), 2)}%\n"
|
||
f"relative_max={format_float(row.get(relative_metric_key.replace('mean_', 'max_')), 2)}%"
|
||
)
|
||
tooltip = "".join(tooltip_lines)
|
||
point_class = "point-top" if is_top else "point-default"
|
||
count_class = "count-top" if is_top else "count-default"
|
||
count_y = max(top + 10.0, y - 6.0)
|
||
abs_point_parts.append(
|
||
f"<circle cx='{x:.1f}' cy='{y:.1f}' r='{point_radius:.1f}' class='{point_class}'>"
|
||
f"<title>{html_escape(tooltip)}</title></circle>"
|
||
)
|
||
abs_point_parts.append(
|
||
f"<text x='{x:.1f}' y='{count_y:.1f}' text-anchor='middle' class='{count_class}'>n={html_escape(str(row['count']))}</text>"
|
||
)
|
||
if show_relative and relative_metric_key:
|
||
relative_value = to_float(row.get(relative_metric_key))
|
||
if relative_value is not None:
|
||
relative_y = scaled_y(float(relative_value), max_relative_value, relative_top, relative_height)
|
||
relative_bar_class = "relative-bar-top" if is_top else "relative-bar"
|
||
relative_point_class = "relative-point-top" if is_top else "relative-point-default"
|
||
relative_bar_parts.append(
|
||
f"<rect x='{x - bar_width / 2.0:.1f}' y='{relative_y:.1f}' width='{bar_width:.1f}' "
|
||
f"height='{max(0.0, relative_bottom - relative_y):.1f}' class='{relative_bar_class}'>"
|
||
f"<title>{html_escape(tooltip)}</title></rect>"
|
||
)
|
||
relative_line_points.append(f"{x:.1f},{relative_y:.1f}")
|
||
relative_point_parts.append(
|
||
f"<circle cx='{x:.1f}' cy='{relative_y:.1f}' r='2.8' class='{relative_point_class}'>"
|
||
f"<title>{html_escape(tooltip)}</title></circle>"
|
||
)
|
||
if idx % label_stride == 0 or is_top:
|
||
label_parts.append(
|
||
f"<text x='{x:.1f}' y='{x_label_y:.1f}' transform='rotate(55 {x:.1f} {x_label_y:.1f})' "
|
||
f"text-anchor='start' class='bin-label'>{html_escape(str(row.get('interval_label', row.get('depth_bin_label', 'n/a'))))}</text>"
|
||
)
|
||
|
||
if len(line_points) >= 2:
|
||
parts.append(f"<polyline points='{' '.join(line_points)}' class='trend-line' />")
|
||
parts.extend(abs_point_parts)
|
||
if show_relative:
|
||
parts.extend(relative_bar_parts)
|
||
if len(relative_line_points) >= 2:
|
||
parts.append(f"<polyline points='{' '.join(relative_line_points)}' class='relative-trend-line' />")
|
||
parts.extend(relative_point_parts)
|
||
parts.extend(label_parts)
|
||
|
||
parts.append(
|
||
f"<text x='{left + plot_width / 2:.1f}' y='{height - 18}' text-anchor='middle' class='axis-title'>{html_escape(x_axis_title)}</text>"
|
||
)
|
||
parts.append(
|
||
f"<text x='18' y='{top + plot_height / 2:.1f}' transform='rotate(-90 18 {top + plot_height / 2:.1f})' "
|
||
f"text-anchor='middle' class='axis-title'>Mean error ({html_escape(unit_suffix.strip())})</text>"
|
||
)
|
||
if show_relative:
|
||
parts.append(
|
||
f"<text x='18' y='{relative_top + relative_height / 2:.1f}' transform='rotate(-90 18 {relative_top + relative_height / 2:.1f})' "
|
||
f"text-anchor='middle' class='axis-title'>{html_escape(relative_y_axis_title)}</text>"
|
||
)
|
||
parts.append("</svg>")
|
||
return "".join(parts)
|
||
|
||
|
||
def render_interval_trend_sections(
|
||
grouped_rows: dict[tuple[int, str], list[dict[str, Any]]],
|
||
metric_key: str,
|
||
top_keys: set[tuple[int, str, float]],
|
||
unit_suffix: str,
|
||
section_title: str,
|
||
x_axis_title: str = "Depth bin",
|
||
relative_metric_key: Optional[str] = None,
|
||
relative_y_axis_title: str = "Relative error (%)",
|
||
relative_reference_label: str = "gt_bin",
|
||
relative_reference_unit: str = "m",
|
||
) -> str:
|
||
if not grouped_rows:
|
||
return (
|
||
f"<section class='table-section'><h3>{html_escape(section_title)}</h3>"
|
||
f"{render_manual_summary_slot(section_title)}<p class='muted'>No interval data.</p></section>"
|
||
)
|
||
|
||
cards = []
|
||
for (cls_id, cls_name), rows in sorted(grouped_rows.items(), key=lambda item: item[0][0]):
|
||
cards.append(
|
||
"<article class='hist-card'>"
|
||
f"<h4>{html_escape(str(cls_name))} <span>id={html_escape(str(cls_id))}</span></h4>"
|
||
f"{render_interval_trend_svg(rows, metric_key=metric_key, top_keys=top_keys, unit_suffix=unit_suffix, x_axis_title=x_axis_title, relative_metric_key=relative_metric_key, relative_y_axis_title=relative_y_axis_title, relative_reference_label=relative_reference_label, relative_reference_unit=relative_reference_unit)}"
|
||
"</article>"
|
||
)
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(section_title)}</h3>"
|
||
f"{render_manual_summary_slot(section_title)}<div class='hist-grid'>{''.join(cards)}</div></section>"
|
||
)
|
||
|
||
|
||
def render_badcase_cards(title: str, records: list[dict[str, Any]], report_path: Path) -> str:
|
||
if not records:
|
||
return (
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} (0)</summary>"
|
||
"<p class='muted'>No saved bad cases in this category.</p></details>"
|
||
)
|
||
|
||
cards = []
|
||
for index, record in enumerate(records, start=1):
|
||
image_path = record.get("visualization")
|
||
if not image_path:
|
||
continue
|
||
image_href = relative_href(report_path, image_path)
|
||
caption = f"{record.get('roi', '')} #{index} {record.get('frame_name', '')} {record.get('cls_name', '')}"
|
||
metrics_rows = [
|
||
("conf / IoU", f"{format_float(to_float(record.get('confidence')), 2)} / {format_float(to_float(record.get('match_iou')), 2)}"),
|
||
("yaw err", f"{format_float(to_float(record.get('yaw_abs_deg')), 1)} deg"),
|
||
(
|
||
"direct / edge yaw err",
|
||
f"{format_float(to_float(record.get('direct_visible_yaw_abs_deg')), 1)} / "
|
||
f"{format_float(to_float(record.get('edge_visible_yaw_abs_deg')), 1)} deg",
|
||
),
|
||
("direct-edge gain", f"{format_float(to_float(record.get('direct_minus_edge_visible_yaw_abs_deg')), 1)} deg"),
|
||
("|gt_x|", f"{format_float(to_float(record.get('gt_lateral_abs_m')), 2)} m"),
|
||
("yaw compare", html_escape(format_bool(record.get("yaw_compare_eligible")))),
|
||
("frame", html_escape(str(record.get("frame_name", "")))),
|
||
]
|
||
metric_html = "".join(
|
||
"<tr><th>{}</th><td>{}</td></tr>".format(html_escape(label), value if label == "yaw compare" else html_escape(value))
|
||
for label, value in metrics_rows
|
||
)
|
||
cards.append(
|
||
"<article class='case-card'>"
|
||
f"<button class='image-button' type='button' data-full-src='{image_href}' data-caption='{html_escape(caption)}'>"
|
||
f"<img loading='lazy' src='{image_href}' alt='{html_escape(caption)}'>"
|
||
"</button>"
|
||
"<div class='case-body'>"
|
||
f"<h4>#{index} {html_escape(str(record.get('cls_name', 'unknown')))}"
|
||
f"<span>{html_escape(str(record.get('frame_name', '')))}</span></h4>"
|
||
"<table class='mini-table'><tbody>"
|
||
f"{metric_html}"
|
||
"</tbody></table>"
|
||
f"<a class='image-link' href='{image_href}' target='_blank' rel='noopener noreferrer'>Open image</a>"
|
||
"</div></article>"
|
||
)
|
||
if not cards:
|
||
return (
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} (0)</summary>"
|
||
"<p class='muted'>No saved bad cases in this category.</p></details>"
|
||
)
|
||
return (
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} ({len(cards)})</summary>"
|
||
f"<div class='badcase-grid'>{''.join(cards)}</div></details>"
|
||
)
|
||
|
||
|
||
def class_badcase_title(metric_name: str, threshold: Optional[float], unit: str, cls_name: str) -> str:
|
||
if threshold is None or not math.isfinite(float(threshold)):
|
||
return f"{metric_name}({cls_name})"
|
||
threshold_text = format_float(float(threshold), 1 if unit == "m" else 0 if float(threshold).is_integer() else 1)
|
||
return f"{metric_name}(err>{threshold_text}{unit}, {cls_name})"
|
||
|
||
|
||
def render_image_section(title: str, image_path: Optional[str], report_path: Path, note: str = "") -> str:
|
||
if not image_path:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>No plot available.</p></section>"
|
||
)
|
||
resolved = Path(image_path)
|
||
if not resolved.is_file():
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}<p class='muted'>Plot file not found.</p></section>"
|
||
)
|
||
image_href = relative_href(report_path, resolved)
|
||
note_html = f"<p class='muted'>{html_escape(note)}</p>" if note else ""
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(title)}</h3>"
|
||
f"{render_manual_summary_slot(title)}{note_html}"
|
||
"<div class='badcase-grid'>"
|
||
"<article class='case-card'>"
|
||
f"<button class='image-button' type='button' data-full-src='{image_href}' data-caption='{html_escape(title)}'>"
|
||
f"<img loading='lazy' src='{image_href}' alt='{html_escape(title)}'>"
|
||
"</button>"
|
||
"<div class='case-body'>"
|
||
f"<h4>{html_escape(title)}</h4>"
|
||
f"<a class='image-link' href='{image_href}' target='_blank' rel='noopener noreferrer'>Open image</a>"
|
||
"</div></article></div></section>"
|
||
)
|
||
|
||
|
||
def render_class_badcase_sections(
|
||
category_name: str,
|
||
threshold: Optional[float],
|
||
unit: str,
|
||
manifest_entries: list[dict[str, Any]],
|
||
report_path: Path,
|
||
) -> str:
|
||
if not manifest_entries:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(category_name)}</h3>"
|
||
f"{render_manual_summary_slot(category_name)}<p class='muted'>No saved bad cases in this category.</p></section>"
|
||
)
|
||
sections = []
|
||
for entry in manifest_entries:
|
||
title = class_badcase_title(category_name, threshold, unit, str(entry.get("cls_name", "unknown")))
|
||
records = load_manifest_records(entry.get("manifest_path", ""))
|
||
sections.append(render_badcase_cards(title, records, report_path))
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(category_name)} By Class</h3>"
|
||
f"{render_manual_summary_slot(f'{category_name} By Class')}<p class='muted'>Each class section is a random reservoir sample capped per class.</p>"
|
||
f"{''.join(sections)}</section>"
|
||
)
|
||
|
||
|
||
def render_interval_badcase_sections(
|
||
section_title: str,
|
||
metric_name: str,
|
||
manifest_entries: list[dict[str, Any]],
|
||
report_path: Path,
|
||
note: str = "",
|
||
) -> str:
|
||
if not manifest_entries:
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(section_title)}</h3>"
|
||
f"{render_manual_summary_slot(section_title)}<p class='muted'>No saved bad cases in this category.</p></section>"
|
||
)
|
||
|
||
sections = []
|
||
for entry in manifest_entries:
|
||
records = load_manifest_records(entry.get("manifest_path", ""))
|
||
title = f"{metric_name} {entry.get('bin_label', 'unknown')}"
|
||
if not records:
|
||
sections.append(
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} (0)</summary>"
|
||
"<p class='muted'>No saved bad cases in this category.</p></details>"
|
||
)
|
||
continue
|
||
cards = []
|
||
for index, record in enumerate(records, start=1):
|
||
image_path = record.get("visualization")
|
||
if not image_path:
|
||
continue
|
||
image_href = relative_href(report_path, image_path)
|
||
caption = f"{record.get('roi', '')} #{index} {record.get('frame_name', '')} {title}"
|
||
metrics_rows = [
|
||
("threshold", html_escape(str(record.get("bin_label", "n/a")))),
|
||
("objects", html_escape(str(record.get("badcase_count", 0)))),
|
||
(
|
||
"max / mean",
|
||
html_escape(
|
||
f"{format_float(to_float(record.get('max_metric_value')), 2)} / {format_float(to_float(record.get('mean_metric_value')), 2)} {record.get('metric_unit', '')}"
|
||
),
|
||
),
|
||
("classes", html_escape(str(record.get("cls_summary", "n/a")))),
|
||
("frame", html_escape(str(record.get("frame_name", "")))),
|
||
]
|
||
metric_html = "".join(f"<tr><th>{html_escape(label)}</th><td>{value}</td></tr>" for label, value in metrics_rows)
|
||
cards.append(
|
||
"<article class='case-card'>"
|
||
f"<button class='image-button' type='button' data-full-src='{image_href}' data-caption='{html_escape(caption)}'>"
|
||
f"<img loading='lazy' src='{image_href}' alt='{html_escape(caption)}'>"
|
||
"</button>"
|
||
"<div class='case-body'>"
|
||
f"<h4>#{index} {html_escape(str(record.get('frame_name', '')))}"
|
||
f"<span>{html_escape(str(record.get('cls_summary', '')))}</span></h4>"
|
||
"<table class='mini-table'><tbody>"
|
||
f"{metric_html}"
|
||
"</tbody></table>"
|
||
f"<a class='image-link' href='{image_href}' target='_blank' rel='noopener noreferrer'>Open image</a>"
|
||
"</div></article>"
|
||
)
|
||
sections.append(
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} ({len(cards)})</summary>"
|
||
f"<div class='badcase-grid'>{''.join(cards)}</div></details>"
|
||
)
|
||
note_html = f"<p class='muted'>{html_escape(note)}</p>" if note else ""
|
||
return (
|
||
f"<section class='gallery-section'><h3>{html_escape(section_title)}</h3>"
|
||
f"{render_manual_summary_slot(section_title)}{note_html}{''.join(sections)}</section>"
|
||
)
|
||
|
||
|
||
def render_2d_badcase_cards(title: str, records: list[dict[str, Any]], report_path: Path) -> str:
|
||
if not records:
|
||
return (
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} (0)</summary>"
|
||
"<p class='muted'>No saved 2D bad cases in this category.</p></details>"
|
||
)
|
||
|
||
cards = []
|
||
for index, record in enumerate(records, start=1):
|
||
image_path = record.get("visualization")
|
||
if not image_path:
|
||
continue
|
||
image_href = relative_href(report_path, image_path)
|
||
caption = f"{record.get('roi', '')} #{index} {record.get('frame_name', '')} {record.get('cls_name', '')} {record.get('kind', '')}"
|
||
metrics_rows = [
|
||
("kind", html_escape(str(record.get("kind", "n/a")))),
|
||
("conf", html_escape(format_float(to_float(record.get("confidence")), 2))),
|
||
("max_iou_any", html_escape(format_float(to_float(record.get("max_iou_any")), 3))),
|
||
("max_iou_same_class", html_escape(format_float(to_float(record.get("max_iou_same_class")), 3))),
|
||
("frame", html_escape(str(record.get("frame_name", "")))),
|
||
]
|
||
metric_html = "".join(f"<tr><th>{html_escape(label)}</th><td>{value}</td></tr>" for label, value in metrics_rows)
|
||
cards.append(
|
||
"<article class='case-card'>"
|
||
f"<button class='image-button' type='button' data-full-src='{image_href}' data-caption='{html_escape(caption)}'>"
|
||
f"<img loading='lazy' src='{image_href}' alt='{html_escape(caption)}'>"
|
||
"</button>"
|
||
"<div class='case-body'>"
|
||
f"<h4>#{index} {html_escape(str(record.get('cls_name', 'unknown')))}"
|
||
f"<span>{html_escape(str(record.get('frame_name', '')))}</span></h4>"
|
||
"<table class='mini-table'><tbody>"
|
||
f"{metric_html}"
|
||
"</tbody></table>"
|
||
f"<a class='image-link' href='{image_href}' target='_blank' rel='noopener noreferrer'>Open image</a>"
|
||
"</div></article>"
|
||
)
|
||
|
||
return (
|
||
f"<details class='badcase-group'><summary>{html_escape(title)} ({len(cards)})</summary>"
|
||
f"<div class='badcase-grid'>{''.join(cards)}</div></details>"
|
||
)
|
||
|
||
|
||
def write_combined_html_report(
|
||
path: Path,
|
||
data_yaml: Path,
|
||
split_path: Path,
|
||
image_root: Path,
|
||
combined_payload: dict[str, Any],
|
||
summary_by_roi: dict[str, Any],
|
||
portrait_payload: Optional[dict[str, Any]] = None,
|
||
portrait_pending: bool = False,
|
||
) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
overview_info_rows = [
|
||
("Dataset YAML", html_escape(str(data_yaml))),
|
||
("Split File", html_escape(str(split_path))),
|
||
("Image Root", html_escape(str(image_root))),
|
||
("Entries", html_escape(str(combined_payload.get("num_entries", 0)))),
|
||
("Elapsed Minutes", html_escape(format_float(to_float(combined_payload.get("elapsed_minutes")), 2))),
|
||
]
|
||
portrait_button_html = ""
|
||
portrait_section_html = ""
|
||
report_title = "Two-ROI Validation Analysis Report"
|
||
report_subtitle = "Interactive HTML summary for localization errors and signed-lateral-bin yaw comparison."
|
||
if portrait_payload:
|
||
portrait_summary = portrait_payload.get("summary") or {}
|
||
portrait_split_label = str(portrait_payload.get("split", "train")).upper()
|
||
overview_info_rows.extend(
|
||
[
|
||
("Portrait Split", html_escape(str(portrait_payload.get("split", "n/a")))),
|
||
("Portrait Vehicles", html_escape(str(int(portrait_summary.get("vehicles", 0) or 0)))),
|
||
("Portrait Objects", html_escape(str(int(portrait_summary.get("mapped_objects", 0) or 0)))),
|
||
("Portrait Summary JSON", html_escape(str(portrait_payload.get("summary_path", "n/a")))),
|
||
]
|
||
)
|
||
portrait_button_html = f"<button class='tab-button' data-tab='portrait'>{html_escape(portrait_split_label)} Portrait</button>"
|
||
portrait_section_html = render_data_portrait_section(portrait_payload)
|
||
report_title = "Two-ROI Validation Analysis Report with Dataset Portrait"
|
||
report_subtitle = "Interactive HTML summary for validation errors plus dataset portrait statistics."
|
||
elif portrait_pending:
|
||
overview_info_rows.append(("Portrait Status", html_escape("building")))
|
||
report_subtitle = "Interactive HTML summary for validation errors. Dataset portrait is still building and this report will refresh when it is ready."
|
||
|
||
overview_rows = []
|
||
for roi_name, payload in summary_by_roi.items():
|
||
overall = payload["overall"]
|
||
diagnostics = payload.get("yaw_compare_diagnostics") or {}
|
||
focus_face = diagnostics.get("focus_face_row") or {}
|
||
overview_rows.append(
|
||
[
|
||
html_escape(roi_name.upper()),
|
||
html_escape(str(overall.get("matched_3d", 0))),
|
||
html_escape(str(overall.get("matched_pos", 0))),
|
||
html_escape(str(diagnostics.get("compare_count", 0))),
|
||
html_escape(format_float(overall.get("yaw_mae_deg"), 2)),
|
||
html_escape(format_float(overall.get("x_abs_mae_m"), 3)),
|
||
html_escape(format_float(overall.get("z_abs_mae_m"), 3)),
|
||
html_escape(format_float(focus_face.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(focus_face.get("edge_based_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(focus_face.get("mean_direct_minus_edge_yaw_deg"), 2)),
|
||
html_escape(format_percent(focus_face.get("yaw_compare_edge_better_rate"))),
|
||
]
|
||
)
|
||
|
||
sections = []
|
||
for roi_name, payload in summary_by_roi.items():
|
||
overall = payload["overall"]
|
||
compare_threshold = float(payload.get("yaw_compare_max_lateral_dist_m", DEFAULT_YAW_COMPARE_MAX_LATERAL_DIST_M))
|
||
diagnostics = payload.get("yaw_compare_diagnostics") or {}
|
||
focus_face = diagnostics.get("focus_face_row") or {}
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
confidence_curve_paths_2d = payload.get("confidence_curve_paths_2d") or {}
|
||
confusion_matrix_plot_path_2d = payload.get("confusion_matrix_plot_path_2d")
|
||
focused_confusion_matrix_plot_path_2d = payload.get("focused_confusion_matrix_plot_path_2d")
|
||
focused_confusion_filter = payload.get("focused_confusion_filter") or {}
|
||
focused_classification_summary_2d = payload.get("focused_classification_summary_2d") or {}
|
||
face_selection_summary = payload.get("face_selection_summary") or {}
|
||
fake_class_summary = payload.get("fake_class_summary") or {}
|
||
occlusion_binary_summary = payload.get("occlusion_binary_summary") or {}
|
||
large_vehicle_compare = payload.get("large_vehicle_compare") or {}
|
||
large_vehicle_compare_summary = large_vehicle_compare.get("summary") or {}
|
||
large_vehicle_compare_rows = large_vehicle_compare.get("class_rows") or []
|
||
large_vehicle_yaw_compare_lateral_rows = large_vehicle_compare.get("yaw_compare_lateral_rows") or []
|
||
horizontal_rows = payload.get("horizontal_interval_rows", [])
|
||
vertical_rows = payload.get("vertical_interval_rows", [])
|
||
yaw_rows = payload.get("yaw_interval_rows", [])
|
||
yaw_heading_rows = payload.get("yaw_heading_interval_rows", [])
|
||
yaw_horizontal_rows = payload.get("yaw_horizontal_interval_rows", [])
|
||
yaw_compare_lateral_rows = payload.get("yaw_compare_lateral_rows", [])
|
||
yaw_compare_lateral_rows_by_face_visibility = payload.get("yaw_compare_lateral_rows_by_face_visibility", {}) or {}
|
||
horizontal_grouped = group_rows_by_class(horizontal_rows)
|
||
vertical_grouped = group_rows_by_class(vertical_rows)
|
||
yaw_grouped = group_rows_by_class(yaw_rows)
|
||
yaw_heading_grouped = group_rows_by_class(yaw_heading_rows)
|
||
yaw_horizontal_grouped = group_rows_by_class(yaw_horizontal_rows)
|
||
horizontal_top_keys = compute_interval_top_keys(horizontal_rows, metric_key="mean_x_abs_m", min_count=0)
|
||
vertical_top_keys = compute_interval_top_keys(vertical_rows, metric_key="mean_z_abs_m", min_count=0)
|
||
yaw_top_keys = compute_interval_top_keys(yaw_rows, metric_key="mean_yaw_abs_deg", min_count=0)
|
||
yaw_heading_top_keys = compute_interval_top_keys(yaw_heading_rows, metric_key="mean_yaw_abs_deg", min_count=0)
|
||
yaw_horizontal_top_keys = compute_interval_top_keys(yaw_horizontal_rows, metric_key="mean_yaw_abs_deg", min_count=0)
|
||
yaw_bad_threshold_deg = to_float(payload.get("yaw_bad_threshold_deg"))
|
||
horizontal_bad_threshold_m = to_float(payload.get("horizontal_bad_threshold_m"))
|
||
vertical_bad_threshold_m = to_float(payload.get("vertical_bad_threshold_m"))
|
||
error_bin_badcases = int(payload.get("error_bin_badcases") or 0)
|
||
error_bin_samples_per_bin = int(payload.get("error_bin_samples_per_bin") or 0)
|
||
metric_rows = [
|
||
("Model", html_escape(payload.get("model_path", "n/a"))),
|
||
("Matched 3D", html_escape(str(overall.get("matched_3d", 0)))),
|
||
("Matched Pos", html_escape(str(overall.get("matched_pos", 0)))),
|
||
("Yaw MAE", html_escape(f"{format_float(overall.get('yaw_mae_deg'), 2)} deg")),
|
||
("Horizontal X MAE", html_escape(f"{format_float(overall.get('x_abs_mae_m'), 3)} m")),
|
||
("Vertical Z MAE", html_escape(f"{format_float(overall.get('z_abs_mae_m'), 3)} m")),
|
||
("2D GT min_wh filter", html_escape(f"{format_float(payload.get('min_wh_px'), 1)} px")),
|
||
("Yaw Compare Range", html_escape(f"gt_x in [{-compare_threshold:.1f}, {compare_threshold:.1f}) m")),
|
||
(
|
||
"Yaw Compare Longitudinal Limit",
|
||
html_escape(f"gt_z < {format_float(payload.get('yaw_compare_max_longitudinal_dist_m', DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M), 1)} m"),
|
||
),
|
||
("Yaw Compare Binning", html_escape(f"{HORIZONTAL_LATERAL_BIN_M:.1f} m signed lateral bins")),
|
||
("Yaw Compare Focus Bucket", html_escape(str(diagnostics.get("focus_bucket", "two-face")))),
|
||
("Yaw Compare Pairs", html_escape(str(diagnostics.get("compare_count", 0)))),
|
||
("Yaw Error Periodicity", html_escape("pi-periodic orientation error")),
|
||
]
|
||
large_vehicle_metric_rows = [
|
||
("Classes", html_escape(str(large_vehicle_compare.get("class_scope", LARGE_VEHICLE_CLASS_SCOPE_TEXT)))),
|
||
("Two-Face Matched 3D", html_escape(str(large_vehicle_compare_summary.get("matched_3d", 0)))),
|
||
("Yaw Compare Pairs", html_escape(str(large_vehicle_compare_summary.get("yaw_compare_count", 0)))),
|
||
("Direct regression MAE", html_escape(f"{format_float(large_vehicle_compare_summary.get('direct_regression_yaw_mae_deg'), 2)} deg")),
|
||
("Edge-based MAE", html_escape(f"{format_float(large_vehicle_compare_summary.get('edge_based_yaw_mae_deg'), 2)} deg")),
|
||
(
|
||
"Mean edge gain",
|
||
render_edge_gain_deg(
|
||
None
|
||
if large_vehicle_compare_summary.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(large_vehicle_compare_summary.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
),
|
||
("Edge better rate", render_rate_vs_half(large_vehicle_compare_summary.get("yaw_compare_edge_better_rate"))),
|
||
("Length compare pairs", html_escape(str(large_vehicle_compare_summary.get("length_compare_count", 0)))),
|
||
("Direct length MAE", html_escape(f"{format_float(large_vehicle_compare_summary.get('direct_regression_length_mae_m'), 3)} m")),
|
||
("Side-edge length MAE", html_escape(f"{format_float(large_vehicle_compare_summary.get('side_edge_length_mae_m'), 3)} m")),
|
||
(
|
||
"Mean side-edge gain",
|
||
render_edge_gain_deg(
|
||
None
|
||
if large_vehicle_compare_summary.get("mean_direct_minus_side_edge_length_m") is None
|
||
else -float(large_vehicle_compare_summary.get("mean_direct_minus_side_edge_length_m")),
|
||
digits=3,
|
||
),
|
||
),
|
||
("Side-edge better rate", render_rate_vs_half(large_vehicle_compare_summary.get("length_compare_edge_better_rate"))),
|
||
]
|
||
compare_rows = [
|
||
["Direct regression MAE", html_escape(f"{format_float(focus_face.get('direct_regression_yaw_mae_deg'), 2)} deg")],
|
||
["Direct regression P90", html_escape(f"{format_float(focus_face.get('direct_regression_yaw_p90_deg'), 2)} deg")],
|
||
["Edge-based MAE", html_escape(f"{format_float(focus_face.get('edge_based_yaw_mae_deg'), 2)} deg")],
|
||
["Edge-based P90", html_escape(f"{format_float(focus_face.get('edge_based_yaw_p90_deg'), 2)} deg")],
|
||
[
|
||
"Mean direct-edge gain",
|
||
html_escape(f"{format_float(focus_face.get('mean_direct_minus_edge_yaw_deg'), 2)} deg"),
|
||
],
|
||
[
|
||
"Median direct-edge gain",
|
||
html_escape(f"{format_float(focus_face.get('median_direct_minus_edge_yaw_deg'), 2)} deg"),
|
||
],
|
||
["Edge better rate", html_escape(format_percent(focus_face.get("yaw_compare_edge_better_rate")))],
|
||
["Direct better rate", html_escape(format_percent(focus_face.get("yaw_compare_direct_better_rate")))],
|
||
["Tie rate", html_escape(format_percent(focus_face.get("yaw_compare_tie_rate")))],
|
||
]
|
||
face_selection_table_rows = [
|
||
[
|
||
html_escape(str(row.get("selection"))),
|
||
html_escape(str(row.get("total", 0))),
|
||
html_escape(str(row.get("correct", 0))),
|
||
html_escape(format_percent(row.get("accuracy"))),
|
||
]
|
||
for row in label_accuracy_rows(face_selection_summary, label_key="selection")
|
||
]
|
||
face_selection_confusion_rows = [
|
||
[html_escape(str(row.get("gt_selection")))]
|
||
+ [html_escape(str(row.get(label, 0))) for label in face_selection_summary.get("label_order", [])]
|
||
for row in label_confusion_matrix_rows(face_selection_summary, label_key="gt_selection")
|
||
]
|
||
fake_class_table_rows = [
|
||
[
|
||
html_escape(str(row.get("fake_class"))),
|
||
html_escape(str(row.get("total", 0))),
|
||
html_escape(str(row.get("correct", 0))),
|
||
html_escape(format_percent(row.get("accuracy"))),
|
||
]
|
||
for row in label_accuracy_rows(fake_class_summary, label_key="fake_class")
|
||
]
|
||
occlusion_binary_table_rows = [
|
||
[
|
||
html_escape(str(row.get("occlusion"))),
|
||
html_escape(str(row.get("total", 0))),
|
||
html_escape(str(row.get("correct", 0))),
|
||
html_escape(format_percent(row.get("accuracy"))),
|
||
]
|
||
for row in label_accuracy_rows(occlusion_binary_summary, label_key="occlusion")
|
||
]
|
||
|
||
class_rows = sorted(
|
||
payload.get("class_rows", []),
|
||
key=lambda row: (int(row.get("two_face_compare_count") or 0), int(row.get("matched_3d") or 0), float(row.get("yaw_abs_sum") or 0.0)),
|
||
reverse=True,
|
||
)
|
||
class_2d_highlights = {
|
||
"cls_acc_2d": top_bad_class_ids(class_rows, "cls_acc_2d", higher_is_worse=False, support_key="cls_eval_pairs_2d"),
|
||
"precision_2d": top_bad_class_ids(class_rows, "precision_2d", higher_is_worse=False, support_key="pred_total"),
|
||
"recall_2d": top_bad_class_ids(class_rows, "recall_2d", higher_is_worse=False, support_key="gt_total"),
|
||
"f1_2d": top_bad_class_ids(class_rows, "f1_2d", higher_is_worse=False, support_key="gt_total"),
|
||
"map50_2d": top_bad_class_ids(class_rows, "map50_2d", higher_is_worse=False, support_key="gt_total"),
|
||
"map50_95_2d": top_bad_class_ids(class_rows, "map50_95_2d", higher_is_worse=False, support_key="gt_total"),
|
||
"false_negatives_2d": top_bad_class_ids(class_rows, "false_negatives_2d", higher_is_worse=True, support_key="gt_total"),
|
||
"false_positives_2d": top_bad_class_ids(class_rows, "false_positives_2d", higher_is_worse=True, support_key="pred_total"),
|
||
}
|
||
class_3d_highlights = {
|
||
"yaw_mae_deg": top_bad_class_ids(class_rows, "yaw_mae_deg", higher_is_worse=True, support_key="matched_3d"),
|
||
"x_abs_mae_m": top_bad_class_ids(class_rows, "x_abs_mae_m", higher_is_worse=True, support_key="matched_pos"),
|
||
"z_abs_mae_m": top_bad_class_ids(class_rows, "z_abs_mae_m", higher_is_worse=True, support_key="matched_pos"),
|
||
}
|
||
class_table_rows = [
|
||
[
|
||
html_escape(str(row.get("cls_id"))),
|
||
html_escape(str(row.get("cls_name"))),
|
||
html_escape(str(row.get("matched_3d", 0))),
|
||
html_escape(str(row.get("two_face_compare_count", 0))),
|
||
highlight_if_needed(
|
||
html_escape(format_float(row.get("yaw_mae_deg"), 2)),
|
||
int(row.get("cls_id", -1)) in class_3d_highlights["yaw_mae_deg"],
|
||
),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
render_edge_gain_deg(
|
||
None
|
||
if row.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(row.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
render_rate_vs_half(row.get("yaw_compare_edge_better_rate")),
|
||
highlight_if_needed(
|
||
html_escape(format_float(row.get("x_abs_mae_m"), 3)),
|
||
int(row.get("cls_id", -1)) in class_3d_highlights["x_abs_mae_m"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_float(row.get("z_abs_mae_m"), 3)),
|
||
int(row.get("cls_id", -1)) in class_3d_highlights["z_abs_mae_m"],
|
||
),
|
||
]
|
||
for row in class_rows
|
||
]
|
||
large_vehicle_class_table_rows = [
|
||
[
|
||
html_escape(str(row.get("cls_id"))),
|
||
html_escape(str(row.get("cls_name"))),
|
||
html_escape(str(row.get("two_face_matched_3d", 0))),
|
||
html_escape(str(row.get("two_face_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
render_edge_gain_deg(
|
||
None
|
||
if row.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(row.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
render_rate_vs_half(row.get("yaw_compare_edge_better_rate")),
|
||
]
|
||
for row in large_vehicle_compare_rows
|
||
]
|
||
class_2d_table_rows = [
|
||
[
|
||
html_escape(str(row.get("cls_id"))),
|
||
html_escape(str(row.get("cls_name"))),
|
||
html_escape(str(row.get("gt_total", 0))),
|
||
html_escape(str(row.get("pred_total", 0))),
|
||
html_escape(str(row.get("matched_2d", 0))),
|
||
html_escape(str(row.get("cls_eval_pairs_2d", 0))),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("cls_acc_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["cls_acc_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("precision_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["precision_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("recall_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["recall_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("f1_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["f1_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("map50_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["map50_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(format_percent(row.get("map50_95_2d"))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["map50_95_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(str(int(row.get("false_negatives_2d", 0)))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["false_negatives_2d"],
|
||
),
|
||
highlight_if_needed(
|
||
html_escape(str(int(row.get("false_positives_2d", 0)))),
|
||
int(row.get("cls_id", -1)) in class_2d_highlights["false_positives_2d"],
|
||
),
|
||
]
|
||
for row in class_rows
|
||
]
|
||
interval_overview_rows = [
|
||
[
|
||
html_escape(str(item.get("cls_id"))),
|
||
html_escape(str(item.get("cls_name"))),
|
||
html_escape(str(item.get("matched_3d", 0))),
|
||
html_escape(str(item.get("yaw_compare_count", item.get("matched_3d", 0)))),
|
||
html_escape(str(item.get("horizontal_bins_text", "n/a"))),
|
||
html_escape(str(item.get("vertical_bins_text", "n/a"))),
|
||
html_escape(str(item.get("yaw_bins_text", "n/a"))),
|
||
]
|
||
for item in payload.get("per_class_interval_insights", [])
|
||
]
|
||
face_visibility_rows = [
|
||
[
|
||
html_escape(str(row.get("key"))),
|
||
html_escape(str(row.get("matched_3d", 0))),
|
||
html_escape(str(row.get("yaw_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
html_escape(format_percent(row.get("yaw_compare_edge_better_rate"))),
|
||
html_escape(str(row.get("length_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_length_mae_m"), 3)),
|
||
html_escape(format_float(row.get("side_edge_length_mae_m"), 3)),
|
||
html_escape(format_percent(row.get("length_compare_edge_better_rate"))),
|
||
]
|
||
for row in diagnostics.get("face_visibility_rows", [])
|
||
]
|
||
yaw_compare_lateral_headers = [
|
||
"lateral_bin",
|
||
"yaw_compare_count",
|
||
"direct_regression_mae_deg",
|
||
"edge_based_mae_deg",
|
||
"edge_gain_deg",
|
||
"edge_better_rate",
|
||
]
|
||
yaw_compare_lateral_table_rows = [
|
||
[
|
||
html_escape(str(row.get("lateral_bin_label"))),
|
||
html_escape(str(row.get("yaw_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
render_edge_gain_deg(
|
||
None
|
||
if row.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(row.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
render_rate_vs_half(row.get("yaw_compare_edge_better_rate")),
|
||
]
|
||
for row in yaw_compare_lateral_rows
|
||
]
|
||
large_vehicle_yaw_compare_lateral_table_rows = [
|
||
[
|
||
html_escape(str(row.get("lateral_bin_label"))),
|
||
html_escape(str(row.get("yaw_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
render_edge_gain_deg(
|
||
None
|
||
if row.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(row.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
render_rate_vs_half(row.get("yaw_compare_edge_better_rate")),
|
||
]
|
||
for row in large_vehicle_yaw_compare_lateral_rows
|
||
]
|
||
yaw_compare_lateral_bucket_sections = [
|
||
render_html_table(
|
||
f"Yaw Comparison by Signed Lateral Bin ({bucket_name})",
|
||
yaw_compare_lateral_headers,
|
||
[
|
||
[
|
||
html_escape(str(row.get("lateral_bin_label"))),
|
||
html_escape(str(row.get("yaw_compare_count", 0))),
|
||
html_escape(format_float(row.get("direct_regression_yaw_mae_deg"), 2)),
|
||
html_escape(format_float(row.get("edge_based_yaw_mae_deg"), 2)),
|
||
render_edge_gain_deg(
|
||
None
|
||
if row.get("mean_direct_minus_edge_yaw_deg") is None
|
||
else -float(row.get("mean_direct_minus_edge_yaw_deg")),
|
||
digits=2,
|
||
),
|
||
render_rate_vs_half(row.get("yaw_compare_edge_better_rate")),
|
||
]
|
||
for row in yaw_compare_lateral_rows_by_face_visibility.get(bucket_name, [])
|
||
],
|
||
)
|
||
for bucket_name in FACE_VISIBILITY_BUCKET_ORDER
|
||
]
|
||
|
||
manifests = {
|
||
category: load_manifest_records(path_str)
|
||
for category, path_str in (payload.get("badcase_manifest_paths") or {}).items()
|
||
}
|
||
error_bin_badcase_entries = payload.get("error_bin_badcase_manifest_entries") or {}
|
||
class_badcase_manifest_entries = payload.get("class_badcase_manifest_paths") or {}
|
||
section_html = [
|
||
f"<section class='tab-panel' id='panel-{html_escape(roi_name)}'>",
|
||
f"<h2>{html_escape(roi_name.upper())}</h2>",
|
||
"<p class='muted'>"
|
||
f"Yaw comparison in this section uses paired samples with <code>gt_x</code> binned over "
|
||
f"<code>[{-compare_threshold:.1f}, {compare_threshold:.1f})m</code> and <code>gt_z < "
|
||
f"{format_float(payload.get('yaw_compare_max_longitudinal_dist_m', DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M), 1)}m</code> in "
|
||
f"<code>{HORIZONTAL_LATERAL_BIN_M:.1f}m</code> signed lateral bins. "
|
||
"Direct-regression and edge-based yaw metrics below are both compared against gt_yaw, with pi-periodic orientation error. "
|
||
"The signed-lateral-bin tables are shown both for all paired samples and split into front_rear_only, side only, and two-face buckets. "
|
||
"The Per-Class 3D Metrics table uses the two-face subset for its direct-vs-edge comparison columns. "
|
||
"Position errors use face-center deltas for face_3d classes and whole-box-center deltas for the remaining classes. "
|
||
f"Detection-dependent report results below use <code>conf > {format_float(payload.get('report_confidence_2d'), 3)}</code> from the ROI F1-confidence sweep "
|
||
f"instead of the configured default <code>{format_float(payload.get('configured_confidence_2d'), 3)}</code>. "
|
||
f"An additional focused 2D confusion matrix below keeps GT objects with <code>|x| < {format_float(focused_confusion_filter.get('max_abs_lateral_m'), 1)}m</code>, "
|
||
f"<code>|z| < {format_float(focused_confusion_filter.get('max_abs_longitudinal_m'), 1)}m</code>, and <code>difficulty={int(focused_confusion_filter.get('required_difficulty', 0) or 0)}</code>. "
|
||
f"The large-vehicle comparison section aggregates <code>{html_escape(str(large_vehicle_compare.get('class_scope', LARGE_VEHICLE_CLASS_SCOPE_TEXT)))}</code> on the same two-face comparison subset."
|
||
"</p>",
|
||
render_metric_value_table("ROI Summary", metric_rows),
|
||
render_metric_value_table(
|
||
"Face/Cut Selection Summary",
|
||
[
|
||
("Eval pairs", html_escape(str(int(face_selection_summary.get("total", 0) or 0)))),
|
||
("Overall accuracy", html_escape(format_percent(face_selection_summary.get("overall_accuracy")))),
|
||
("Mean per-selection accuracy", html_escape(format_percent(face_selection_summary.get("mean_accuracy")))),
|
||
("CSV", html_escape(str(payload.get("face_selection_csv", "n/a")))),
|
||
("Confusion CSV", html_escape(str(payload.get("face_selection_confusion_csv", "n/a")))),
|
||
],
|
||
),
|
||
render_html_table(
|
||
"Face/Cut Selection Accuracy",
|
||
["selection", "total", "correct", "accuracy"],
|
||
face_selection_table_rows,
|
||
),
|
||
render_html_table(
|
||
"Face/Cut Selection Confusion Matrix",
|
||
["gt \\ pred", *[html_escape(str(label)) for label in face_selection_summary.get("label_order", [])]],
|
||
face_selection_confusion_rows,
|
||
),
|
||
render_metric_value_table(
|
||
"Fake Class Classification Summary",
|
||
[
|
||
("Eval pairs", html_escape(str(int(fake_class_summary.get("total", 0) or 0)))),
|
||
("Overall accuracy", html_escape(format_percent(fake_class_summary.get("overall_accuracy")))),
|
||
("Mean class accuracy", html_escape(format_percent(fake_class_summary.get("mean_accuracy")))),
|
||
("Fake vs non-fake bad cases", html_escape(str(int(fake_class_summary.get("fake_vs_non_fake", 0) or 0)))),
|
||
("Fake internal bad cases", html_escape(str(int(fake_class_summary.get("fake_internal_confusion", 0) or 0)))),
|
||
("Visible internal bad cases", html_escape(str(int(fake_class_summary.get("visible_internal_confusion", 0) or 0)))),
|
||
("CSV", html_escape(str(payload.get("fake_class_csv", "n/a")))),
|
||
],
|
||
),
|
||
render_html_table(
|
||
"Fake Class Classification Accuracy",
|
||
["fake_class", "total", "correct", "accuracy"],
|
||
fake_class_table_rows,
|
||
),
|
||
render_metric_value_table(
|
||
"Occlusion Binary Classification Summary",
|
||
[
|
||
("Eval pairs", html_escape(str(int(occlusion_binary_summary.get("total", 0) or 0)))),
|
||
("Overall accuracy", html_escape(format_percent(occlusion_binary_summary.get("overall_accuracy")))),
|
||
("Mean class accuracy", html_escape(format_percent(occlusion_binary_summary.get("mean_accuracy")))),
|
||
("CSV", html_escape(str(payload.get("occlusion_binary_csv", "n/a")))),
|
||
],
|
||
),
|
||
render_html_table(
|
||
"Occlusion Binary Classification Accuracy",
|
||
["occlusion", "total", "correct", "accuracy"],
|
||
occlusion_binary_table_rows,
|
||
),
|
||
render_metric_value_table(
|
||
"2D Threshold Advice",
|
||
[
|
||
("Configured 2D conf", html_escape(format_float(payload.get("configured_confidence_2d"), 3))),
|
||
("Recommended 2D conf", html_escape(format_float(threshold_advice_2d.get("recommended_confidence"), 3))),
|
||
("Advice source", html_escape(str(threshold_advice_2d.get("source", "n/a")))),
|
||
("Mean class precision", html_escape(format_percent(threshold_advice_2d.get("mean_precision")))),
|
||
("Mean class recall", html_escape(format_percent(threshold_advice_2d.get("mean_recall")))),
|
||
("Mean class F1", html_escape(format_percent(threshold_advice_2d.get("mean_f1")))),
|
||
],
|
||
),
|
||
render_metric_value_table(
|
||
"2D Summary",
|
||
[
|
||
("GT total", html_escape(str(overall.get("gt_total", 0)))),
|
||
("Pred total", html_escape(str(overall.get("pred_total", 0)))),
|
||
("Matched 2D", html_escape(str(overall.get("matched_2d", 0)))),
|
||
("2D confidence threshold", html_escape(format_float(threshold_advice_2d.get("recommended_confidence"), 3))),
|
||
("2D cls eval pairs", html_escape(str(int(overall.get("cls_eval_pairs_2d", 0) or 0)))),
|
||
("2D cls acc", html_escape(format_percent(overall.get("cls_acc_2d")))),
|
||
("2D precision", html_escape(format_percent(overall.get("precision_2d")))),
|
||
("2D recall", html_escape(format_percent(overall.get("recall_2d")))),
|
||
("2D F1", html_escape(format_percent(overall.get("f1_2d")))),
|
||
("2D mAP50", html_escape(format_percent(overall.get("map50_2d")))),
|
||
("2D mAP50-95", html_escape(format_percent(overall.get("map50_95_2d")))),
|
||
("Focused GT count", html_escape(str(int(focused_confusion_filter.get("gt_count", 0) or 0)))),
|
||
("Focused 2D cls eval pairs", html_escape(str(int(focused_classification_summary_2d.get("overall_pairs", 0) or 0)))),
|
||
("Focused 2D cls acc", html_escape(format_percent(focused_classification_summary_2d.get("overall_accuracy")))),
|
||
("2D false negatives", html_escape(str(max(0, int(overall.get("gt_total", 0)) - int(overall.get("matched_2d", 0)))))),
|
||
("2D false positives", html_escape(str(max(0, int(overall.get("pred_total", 0)) - int(overall.get("matched_2d", 0)))))),
|
||
],
|
||
),
|
||
render_html_table(
|
||
"Per-Class 2D Metrics",
|
||
[
|
||
"cls_id",
|
||
"cls_name",
|
||
"gt_total",
|
||
"pred_total",
|
||
"matched_2d",
|
||
"cls_eval_pairs_2d",
|
||
"cls_acc_2d",
|
||
"precision_2d",
|
||
"recall_2d",
|
||
"f1_2d",
|
||
"mAP50",
|
||
"mAP50-95",
|
||
"false_negatives",
|
||
"false_positives",
|
||
],
|
||
class_2d_table_rows,
|
||
),
|
||
render_image_section(
|
||
"2D F1-Confidence Curve",
|
||
confidence_curve_paths_2d.get("f1_curve"),
|
||
path,
|
||
note=(
|
||
f"Report threshold is {format_float(payload.get('report_confidence_2d'), 3)}, "
|
||
f"where mean class F1 reaches {format_percent(threshold_advice_2d.get('mean_f1'))}."
|
||
),
|
||
),
|
||
render_image_section(
|
||
"2D Confusion Matrix",
|
||
confusion_matrix_plot_path_2d,
|
||
path,
|
||
note=(
|
||
f"Stats in this matrix use conf > {format_float(payload.get('report_confidence_2d'), 3)}. "
|
||
"Class-agnostic IoU>=0.5 matching, normalized by true class. Background row/column indicates false positives and false negatives."
|
||
),
|
||
),
|
||
render_image_section(
|
||
"2D Confusion Matrix (Easy Near-Range, No Occlusion)",
|
||
focused_confusion_matrix_plot_path_2d,
|
||
path,
|
||
note=(
|
||
f"Focused subset keeps GT objects with |lateral|<{format_float(focused_confusion_filter.get('max_abs_lateral_m'), 1)}m, "
|
||
f"|longitudinal|<{format_float(focused_confusion_filter.get('max_abs_longitudinal_m'), 1)}m, and difficulty="
|
||
f"{int(focused_confusion_filter.get('required_difficulty', 0) or 0)}. "
|
||
"Unmatched FP counts only include predictions whose predicted centers fall inside the same spatial window."
|
||
),
|
||
),
|
||
"<section class='gallery-section'><h3>2D Bad Cases</h3>"
|
||
"<p class='muted'>Displayed samples are random reservoir samples from unmatched GT boxes and unmatched predictions after the IoU>=0.5 greedy matching step.</p>",
|
||
render_2d_badcase_cards("2D False Negative", manifests.get("2d_false_negative", []), path),
|
||
render_2d_badcase_cards("2D False Positive", manifests.get("2d_false_positive", []), path),
|
||
render_2d_badcase_cards("Fake Class Classification", manifests.get("fake_class", []), path),
|
||
render_badcase_cards("Face/Cut Selection", manifests.get("face_selection", []), path),
|
||
"</section>",
|
||
render_html_table("Focused Two-Face Yaw Comparison", ["Metric", "Value"], compare_rows),
|
||
render_metric_value_table("Large-Vehicle Two-Face Comparison", large_vehicle_metric_rows),
|
||
render_html_table(
|
||
"Large-Vehicle Per-Class Comparison",
|
||
[
|
||
"cls_id",
|
||
"cls_name",
|
||
"two_face_matched_3d",
|
||
"two_face_compare_count",
|
||
"direct_regression_mae_deg",
|
||
"edge_based_mae_deg",
|
||
"edge_gain_deg",
|
||
"edge_better_rate",
|
||
],
|
||
large_vehicle_class_table_rows,
|
||
),
|
||
render_html_table(
|
||
"Large-Vehicle Yaw Comparison by Signed Lateral Bin",
|
||
yaw_compare_lateral_headers,
|
||
large_vehicle_yaw_compare_lateral_table_rows,
|
||
),
|
||
render_html_table(
|
||
"Yaw Comparison by Signed Lateral Bin (All Paired Samples)",
|
||
yaw_compare_lateral_headers,
|
||
yaw_compare_lateral_table_rows,
|
||
),
|
||
*yaw_compare_lateral_bucket_sections,
|
||
render_html_table(
|
||
"Face-Visibility Bucket Diagnostics",
|
||
[
|
||
"bucket",
|
||
"matched_3d",
|
||
"yaw_compare_count",
|
||
"direct_regression_mae_deg",
|
||
"edge_based_mae_deg",
|
||
"edge_better_rate",
|
||
"length_compare_count",
|
||
"direct_length_mae_m",
|
||
"side_edge_length_mae_m",
|
||
"side_edge_better_rate",
|
||
],
|
||
face_visibility_rows,
|
||
),
|
||
render_html_table(
|
||
"Per-Class 3D Metrics",
|
||
[
|
||
"cls_id",
|
||
"cls_name",
|
||
"matched_3d",
|
||
"two_face_compare_count",
|
||
"yaw_mae_deg",
|
||
"direct_regression_mae_deg",
|
||
"edge_based_mae_deg",
|
||
"edge_gain_deg",
|
||
"edge_better_rate",
|
||
"x_mae_m",
|
||
"z_mae_m",
|
||
],
|
||
class_table_rows,
|
||
),
|
||
render_html_table(
|
||
"Depth Interval Overview",
|
||
["cls_id", "cls_name", "matched_3d", "yaw_compare_count", "worst horizontal bins", "worst vertical bins", "worst yaw bins"],
|
||
interval_overview_rows,
|
||
),
|
||
"<section class='gallery-section'><h3>All Bin Trend Plots</h3>"
|
||
"<p class='muted'>All bins are displayed below as trend plots. The upper line shows mean absolute error by bin and each point prints the bin sample count. For plots with a lower panel, the semi-transparent histogram shows relative error percentage: horizontal and vertical panels use error divided by the gt-bin center, while the GT-yaw panel uses error divided by |gt_yaw|. The top 3 mean-error bins per class are highlighted in red.</p></section>",
|
||
render_interval_trend_sections(
|
||
horizontal_grouped,
|
||
metric_key="mean_x_abs_m",
|
||
top_keys=horizontal_top_keys,
|
||
unit_suffix="m",
|
||
section_title="Horizontal X Error by 5m Lateral Bin",
|
||
x_axis_title="Lateral bin",
|
||
relative_metric_key="mean_x_abs_pct",
|
||
),
|
||
render_interval_trend_sections(
|
||
vertical_grouped,
|
||
metric_key="mean_z_abs_m",
|
||
top_keys=vertical_top_keys,
|
||
unit_suffix="m",
|
||
section_title="Vertical Z Error by 5m Depth Bin",
|
||
x_axis_title="Depth bin",
|
||
relative_metric_key="mean_z_abs_pct",
|
||
),
|
||
render_interval_trend_sections(
|
||
yaw_grouped,
|
||
metric_key="mean_yaw_abs_deg",
|
||
top_keys=yaw_top_keys,
|
||
unit_suffix="deg",
|
||
section_title="Yaw Orientation Error by 10m Depth Bin",
|
||
x_axis_title="Depth bin",
|
||
),
|
||
render_interval_trend_sections(
|
||
yaw_heading_grouped,
|
||
metric_key="mean_yaw_abs_deg",
|
||
top_keys=yaw_heading_top_keys,
|
||
unit_suffix="deg",
|
||
section_title="Yaw Orientation Error by 10 degree Bin",
|
||
x_axis_title="GT yaw bin",
|
||
relative_metric_key="mean_yaw_abs_pct",
|
||
relative_y_axis_title="Relative yaw error / |gt_yaw| (%)",
|
||
relative_reference_label="mean |gt_yaw|",
|
||
relative_reference_unit="deg",
|
||
),
|
||
render_interval_trend_sections(
|
||
yaw_horizontal_grouped,
|
||
metric_key="mean_yaw_abs_deg",
|
||
top_keys=yaw_horizontal_top_keys,
|
||
unit_suffix="deg",
|
||
section_title="Yaw Orientation Error by 5m Horizontal Bin",
|
||
x_axis_title="Horizontal bin",
|
||
),
|
||
render_interval_badcase_sections(
|
||
"Yaw By Error",
|
||
"Yaw Error",
|
||
error_bin_badcase_entries.get("yaw", []),
|
||
path,
|
||
note=(
|
||
f"Each bin keeps up to {error_bin_badcases} object candidates, then exports up to {error_bin_samples_per_bin} frame-level samples. "
|
||
f"Each image overlays all objects in that frame whose yaw error is at least the bin lower bound ({format_float(ERROR_YAW_BIN_DEG, 1)}deg bins)."
|
||
),
|
||
),
|
||
render_interval_badcase_sections(
|
||
"Horizontal By Error",
|
||
"Horizontal Error",
|
||
error_bin_badcase_entries.get("horizontal", []),
|
||
path,
|
||
note=(
|
||
f"Each bin keeps up to {error_bin_badcases} object candidates, then exports up to {error_bin_samples_per_bin} frame-level samples. "
|
||
f"Each image overlays all objects in that frame whose horizontal error is at least the bin lower bound ({format_float(ERROR_DISTANCE_BIN_M, 1)}m bins)."
|
||
),
|
||
),
|
||
render_interval_badcase_sections(
|
||
"Vertical By Error",
|
||
"Vertical Error",
|
||
error_bin_badcase_entries.get("vertical", []),
|
||
path,
|
||
note=(
|
||
f"Each bin keeps up to {error_bin_badcases} object candidates, then exports up to {error_bin_samples_per_bin} frame-level samples. "
|
||
f"Each image overlays all objects in that frame whose vertical error is at least the bin lower bound ({format_float(ERROR_DISTANCE_BIN_M, 1)}m bins)."
|
||
),
|
||
),
|
||
render_class_badcase_sections("Yaw", yaw_bad_threshold_deg, "deg", class_badcase_manifest_entries.get("yaw", []), path),
|
||
render_class_badcase_sections(
|
||
"Horizontal", horizontal_bad_threshold_m, "m", class_badcase_manifest_entries.get("horizontal", []), path
|
||
),
|
||
render_class_badcase_sections("Vertical", vertical_bad_threshold_m, "m", class_badcase_manifest_entries.get("vertical", []), path),
|
||
"</section>",
|
||
]
|
||
sections.append("".join(section_html))
|
||
|
||
html_text = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>Two-ROI Validation Analysis Report</title>
|
||
<style>
|
||
:root {{
|
||
--bg: #f6f5ef;
|
||
--panel: #fffdf7;
|
||
--ink: #1f2328;
|
||
--muted: #5b6470;
|
||
--line: #d8d2c4;
|
||
--accent: #0e7490;
|
||
--accent-soft: #d7eef4;
|
||
--shadow: 0 12px 30px rgba(31, 35, 40, 0.08);
|
||
}}
|
||
* {{ box-sizing: border-box; }}
|
||
body {{
|
||
margin: 0;
|
||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||
color: var(--ink);
|
||
background: linear-gradient(180deg, #f2efe6 0%, var(--bg) 24%, #fbfaf6 100%);
|
||
}}
|
||
header {{
|
||
padding: 32px 28px 18px;
|
||
border-bottom: 1px solid var(--line);
|
||
background: rgba(255, 253, 247, 0.92);
|
||
position: sticky;
|
||
top: 0;
|
||
backdrop-filter: blur(12px);
|
||
z-index: 20;
|
||
}}
|
||
h1, h2, h3, h4 {{ margin: 0 0 12px; }}
|
||
p {{ margin: 0 0 10px; line-height: 1.55; }}
|
||
code {{
|
||
background: #efe8d7;
|
||
padding: 2px 6px;
|
||
border-radius: 6px;
|
||
font-size: 0.95em;
|
||
}}
|
||
.emphasis-red {{
|
||
color: #b42318;
|
||
font-weight: 700;
|
||
}}
|
||
.emphasis-green {{
|
||
color: #067647;
|
||
font-weight: 700;
|
||
}}
|
||
.muted {{ color: var(--muted); }}
|
||
.manual-summary {{
|
||
min-height: 72px;
|
||
border: 2px dashed #c7bda9;
|
||
border-radius: 14px;
|
||
background: #fffaf0;
|
||
margin: 12px 0 14px;
|
||
position: relative;
|
||
padding: 34px 14px 12px;
|
||
white-space: pre-wrap;
|
||
}}
|
||
.manual-summary::before {{
|
||
content: "Manual Summary";
|
||
position: absolute;
|
||
top: 10px;
|
||
left: 12px;
|
||
color: #9a8f78;
|
||
font-size: 12px;
|
||
font-weight: 700;
|
||
letter-spacing: 0.04em;
|
||
text-transform: uppercase;
|
||
}}
|
||
.tabs {{
|
||
display: flex;
|
||
gap: 10px;
|
||
flex-wrap: wrap;
|
||
margin-top: 16px;
|
||
}}
|
||
.tab-button {{
|
||
border: 1px solid var(--line);
|
||
background: #fff9ec;
|
||
color: var(--ink);
|
||
padding: 10px 14px;
|
||
border-radius: 999px;
|
||
cursor: pointer;
|
||
font-weight: 600;
|
||
}}
|
||
.tab-button.active {{
|
||
background: var(--accent);
|
||
color: white;
|
||
border-color: var(--accent);
|
||
}}
|
||
main {{
|
||
padding: 24px 24px 56px;
|
||
max-width: 1680px;
|
||
margin: 0 auto;
|
||
}}
|
||
.tab-panel {{
|
||
display: none;
|
||
gap: 18px;
|
||
margin-top: 18px;
|
||
}}
|
||
.tab-panel.active {{ display: block; }}
|
||
.table-section, .gallery-section {{
|
||
background: var(--panel);
|
||
border: 1px solid var(--line);
|
||
border-radius: 18px;
|
||
padding: 18px;
|
||
margin-bottom: 18px;
|
||
box-shadow: var(--shadow);
|
||
}}
|
||
.table-wrap {{
|
||
overflow-x: auto;
|
||
margin-top: 12px;
|
||
}}
|
||
table {{
|
||
width: 100%;
|
||
border-collapse: collapse;
|
||
font-size: 14px;
|
||
min-width: 720px;
|
||
}}
|
||
th, td {{
|
||
padding: 10px 12px;
|
||
border-bottom: 1px solid var(--line);
|
||
text-align: left;
|
||
vertical-align: top;
|
||
}}
|
||
th {{
|
||
background: #f2eee2;
|
||
position: sticky;
|
||
top: 0;
|
||
z-index: 1;
|
||
}}
|
||
.metric-table {{ min-width: 480px; }}
|
||
.metric-table th, .metric-table td {{ width: 50%; }}
|
||
.badcase-group {{
|
||
margin-top: 14px;
|
||
border: 1px solid var(--line);
|
||
border-radius: 14px;
|
||
background: #fffaf0;
|
||
overflow: hidden;
|
||
}}
|
||
.badcase-group summary {{
|
||
cursor: pointer;
|
||
list-style: none;
|
||
padding: 14px 16px;
|
||
font-weight: 700;
|
||
background: #f2eee2;
|
||
}}
|
||
.badcase-grid {{
|
||
display: grid;
|
||
grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
|
||
gap: 16px;
|
||
padding: 16px;
|
||
}}
|
||
.hist-grid {{
|
||
display: grid;
|
||
grid-template-columns: 1fr;
|
||
gap: 16px;
|
||
}}
|
||
.hist-card {{
|
||
border: 1px solid var(--line);
|
||
border-radius: 16px;
|
||
background: white;
|
||
padding: 14px;
|
||
overflow-x: auto;
|
||
box-shadow: var(--shadow);
|
||
}}
|
||
.hist-card h4 {{
|
||
display: flex;
|
||
justify-content: space-between;
|
||
gap: 12px;
|
||
margin-bottom: 12px;
|
||
font-size: 15px;
|
||
}}
|
||
.histogram-svg {{
|
||
display: block;
|
||
width: 100%;
|
||
min-width: 720px;
|
||
height: auto;
|
||
}}
|
||
.axis-line {{
|
||
stroke: #8e8a81;
|
||
stroke-width: 1;
|
||
}}
|
||
.grid-line {{
|
||
stroke: #ece5d7;
|
||
stroke-width: 1;
|
||
}}
|
||
.grid-line-minor {{
|
||
stroke: #f3ede2;
|
||
stroke-width: 0.8;
|
||
}}
|
||
.axis-text {{
|
||
fill: var(--muted);
|
||
font-size: 11px;
|
||
}}
|
||
.axis-text-minor {{
|
||
fill: #8b8376;
|
||
font-size: 9px;
|
||
}}
|
||
.axis-title {{
|
||
fill: var(--ink);
|
||
font-size: 12px;
|
||
font-weight: 600;
|
||
}}
|
||
.bin-label {{
|
||
fill: var(--muted);
|
||
font-size: 10px;
|
||
}}
|
||
.trend-line {{
|
||
fill: none;
|
||
stroke: #0e7490;
|
||
stroke-width: 2;
|
||
stroke-linejoin: round;
|
||
stroke-linecap: round;
|
||
opacity: 0.82;
|
||
}}
|
||
.relative-bar {{
|
||
fill: #0e7490;
|
||
opacity: 0.22;
|
||
}}
|
||
.relative-bar-top {{
|
||
fill: #b42318;
|
||
opacity: 0.28;
|
||
}}
|
||
.relative-trend-line {{
|
||
fill: none;
|
||
stroke: #155eef;
|
||
stroke-width: 1.8;
|
||
stroke-linejoin: round;
|
||
stroke-linecap: round;
|
||
opacity: 0.88;
|
||
}}
|
||
.relative-point-default {{
|
||
fill: #155eef;
|
||
stroke: #fffdf7;
|
||
stroke-width: 1.0;
|
||
}}
|
||
.relative-point-top {{
|
||
fill: #b42318;
|
||
stroke: #fffdf7;
|
||
stroke-width: 1.0;
|
||
}}
|
||
.point-default {{
|
||
fill: #0e7490;
|
||
stroke: #fffdf7;
|
||
stroke-width: 1.2;
|
||
}}
|
||
.point-top {{
|
||
fill: #b42318;
|
||
stroke: #fffdf7;
|
||
stroke-width: 1.2;
|
||
}}
|
||
.count-default {{
|
||
fill: var(--muted);
|
||
font-size: 9px;
|
||
}}
|
||
.count-top {{
|
||
fill: #b42318;
|
||
font-size: 9px;
|
||
font-weight: 700;
|
||
}}
|
||
.case-card {{
|
||
border: 1px solid var(--line);
|
||
border-radius: 16px;
|
||
background: white;
|
||
overflow: hidden;
|
||
box-shadow: var(--shadow);
|
||
}}
|
||
.image-button {{
|
||
display: block;
|
||
width: 100%;
|
||
padding: 0;
|
||
margin: 0;
|
||
border: 0;
|
||
background: #16181b;
|
||
cursor: zoom-in;
|
||
}}
|
||
.image-button img {{
|
||
display: block;
|
||
width: 100%;
|
||
height: auto;
|
||
}}
|
||
.case-body {{
|
||
padding: 14px 14px 16px;
|
||
}}
|
||
.case-body h4 {{
|
||
display: flex;
|
||
justify-content: space-between;
|
||
gap: 12px;
|
||
font-size: 15px;
|
||
margin-bottom: 12px;
|
||
}}
|
||
.mini-table {{
|
||
width: 100%;
|
||
min-width: 0;
|
||
font-size: 13px;
|
||
}}
|
||
.mini-table th, .mini-table td {{
|
||
padding: 6px 0;
|
||
border-bottom: 1px dashed var(--line);
|
||
}}
|
||
.mini-table th {{
|
||
position: static;
|
||
background: transparent;
|
||
width: 40%;
|
||
color: var(--muted);
|
||
font-weight: 600;
|
||
}}
|
||
.image-link {{
|
||
display: inline-block;
|
||
margin-top: 10px;
|
||
color: var(--accent);
|
||
text-decoration: none;
|
||
font-weight: 600;
|
||
}}
|
||
.image-link:hover {{ text-decoration: underline; }}
|
||
.modal {{
|
||
position: fixed;
|
||
inset: 0;
|
||
background: rgba(13, 16, 20, 0.82);
|
||
display: none;
|
||
align-items: center;
|
||
justify-content: center;
|
||
padding: 24px;
|
||
z-index: 50;
|
||
}}
|
||
.modal.open {{ display: flex; }}
|
||
.modal-content {{
|
||
max-width: min(94vw, 1600px);
|
||
max-height: 94vh;
|
||
}}
|
||
.modal-content img {{
|
||
width: 100%;
|
||
height: auto;
|
||
max-height: 84vh;
|
||
object-fit: contain;
|
||
border-radius: 14px;
|
||
box-shadow: var(--shadow);
|
||
}}
|
||
.modal-caption {{
|
||
color: white;
|
||
margin-top: 10px;
|
||
text-align: center;
|
||
}}
|
||
@media (max-width: 960px) {{
|
||
main {{ padding: 18px 14px 40px; }}
|
||
header {{ padding: 24px 16px 16px; }}
|
||
.badcase-grid {{ grid-template-columns: 1fr; }}
|
||
}}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<header>
|
||
<h1>{html_escape(report_title)}</h1>
|
||
<p class="muted">{html_escape(report_subtitle)}</p>
|
||
<div class="tabs">
|
||
<button class="tab-button active" data-tab="overview">Overview</button>
|
||
{portrait_button_html}
|
||
{''.join(f"<button class='tab-button' data-tab='panel-{html_escape(roi_name)}'>{html_escape(roi_name.upper())}</button>" for roi_name in summary_by_roi)}
|
||
</div>
|
||
</header>
|
||
<main>
|
||
<section class="tab-panel active" id="overview">
|
||
{render_metric_value_table("Run Info", overview_info_rows)}
|
||
{render_html_table(
|
||
"ROI Overview",
|
||
["ROI", "matched_3d", "matched_pos", "yaw_compare_count", "yaw_mae_deg", "x_mae_m", "z_mae_m", "direct_regression_mae_deg", "edge_based_mae_deg", "direct-edge_gain_deg", "edge_better_rate"],
|
||
overview_rows,
|
||
)}
|
||
</section>
|
||
{portrait_section_html}
|
||
{''.join(sections)}
|
||
</main>
|
||
<div class="modal" id="image-modal" role="dialog" aria-modal="true">
|
||
<div class="modal-content">
|
||
<img id="modal-image" src="" alt="">
|
||
<div class="modal-caption" id="modal-caption"></div>
|
||
</div>
|
||
</div>
|
||
<script>
|
||
const tabButtons = Array.from(document.querySelectorAll('.tab-button'));
|
||
const panels = Array.from(document.querySelectorAll('.tab-panel'));
|
||
tabButtons.forEach((button) => {{
|
||
button.addEventListener('click', () => {{
|
||
const target = button.dataset.tab;
|
||
tabButtons.forEach((item) => item.classList.toggle('active', item === button));
|
||
panels.forEach((panel) => panel.classList.toggle('active', panel.id === target));
|
||
window.scrollTo({{ top: 0, behavior: 'smooth' }});
|
||
}});
|
||
}});
|
||
|
||
const modal = document.getElementById('image-modal');
|
||
const modalImage = document.getElementById('modal-image');
|
||
const modalCaption = document.getElementById('modal-caption');
|
||
document.querySelectorAll('.image-button').forEach((button) => {{
|
||
button.addEventListener('click', () => {{
|
||
modalImage.src = button.dataset.fullSrc;
|
||
modalCaption.textContent = button.dataset.caption || '';
|
||
modal.classList.add('open');
|
||
}});
|
||
}});
|
||
modal.addEventListener('click', () => modal.classList.remove('open'));
|
||
document.addEventListener('keydown', (event) => {{
|
||
if (event.key === 'Escape') {{
|
||
modal.classList.remove('open');
|
||
}}
|
||
}});
|
||
</script>
|
||
</body>
|
||
</html>
|
||
"""
|
||
path.write_text(html_text, encoding="utf-8")
|
||
|
||
|
||
def run_roi_analysis(
|
||
bundle,
|
||
entries: list[tuple[str, str]],
|
||
image_root: Path,
|
||
class_map: dict[str, int],
|
||
difficulty_weights: list[float],
|
||
face_3d_classes: set[int],
|
||
complete_3d_classes: set[int],
|
||
classes: Optional[set[int]],
|
||
min_wh_px: float,
|
||
face_visibility_score_thresh: float,
|
||
yaw_bad_threshold_deg: float,
|
||
edge_yaw_max_lateral_dist_m: float,
|
||
yaw_compare_max_lateral_dist_m: float,
|
||
horizontal_bad_threshold_m: float,
|
||
vertical_bad_threshold_m: float,
|
||
topk_badcases: int,
|
||
per_class_badcases: int,
|
||
error_bin_badcases: int,
|
||
error_bin_samples_per_bin: int,
|
||
badcase_random_seed: int,
|
||
output_root: Path,
|
||
log_every: int,
|
||
split_file: str,
|
||
inference_batch_size: int,
|
||
) -> dict[str, Any]:
|
||
roi_name = bundle.spec.name.lower()
|
||
roi_output = output_root / roi_name
|
||
roi_output.mkdir(parents=True, exist_ok=True)
|
||
edge_yaw_compare_score_thresh = float(EDGE_YAW_VALID_VISIBILITY_SCORE_THRESH)
|
||
focused_confusion_max_abs_longitudinal_m = focused_confusion_max_abs_longitudinal_m_for_roi(roi_name)
|
||
configured_confidence_2d = float(bundle.spec.conf)
|
||
threshold_search_conf = min(float(configured_confidence_2d), float(MIN_CONFIDENCE_FOR_2D_THRESHOLD_SEARCH))
|
||
|
||
bad_yaw_handle, bad_yaw_writer = make_writer(roi_output / "bad_cases_yaw.csv")
|
||
bad_x_handle, bad_x_writer = make_writer(roi_output / "bad_cases_horizontal.csv")
|
||
bad_y_handle, bad_y_writer = make_writer(roi_output / "bad_cases_vertical.csv")
|
||
bad_face_selection_handle, bad_face_selection_writer = make_writer(
|
||
roi_output / "bad_cases_face_selection.csv",
|
||
fieldnames=FACE_SELECTION_BADCASE_FIELDS,
|
||
)
|
||
|
||
names_dict = names_to_dict(bundle.names)
|
||
ap_summary_2d, threshold_advice_2d, report_confidence_2d = collect_2d_confidence_advice(
|
||
bundle=bundle,
|
||
entries=entries,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
classes=classes,
|
||
min_wh_px=min_wh_px,
|
||
names_dict=names_dict,
|
||
roi_name=roi_name,
|
||
roi_output=roi_output,
|
||
topk_badcases=topk_badcases,
|
||
badcase_random_seed=badcase_random_seed,
|
||
focused_confusion_max_abs_longitudinal_m=focused_confusion_max_abs_longitudinal_m,
|
||
threshold_search_conf=threshold_search_conf,
|
||
configured_confidence_2d=configured_confidence_2d,
|
||
inference_batch_size=inference_batch_size,
|
||
log_every=log_every,
|
||
)
|
||
|
||
overall_bucket = make_metric_bucket()
|
||
class_buckets = defaultdict(make_metric_bucket)
|
||
detailed_class_buckets = defaultdict(make_metric_bucket)
|
||
class_face_visibility_buckets = defaultdict(lambda: defaultdict(make_metric_bucket))
|
||
detailed_class_face_visibility_buckets = defaultdict(lambda: defaultdict(make_metric_bucket))
|
||
two_d_eval_packets: list[dict[str, Any]] = []
|
||
face_selection_store = make_label_accuracy_store(FACE_SELECTION_LABEL_ORDER)
|
||
occlusion_binary_store = make_label_accuracy_store(OCCLUSION_BINARY_LABEL_ORDER)
|
||
breakdowns = make_breakdown_buckets()
|
||
horizontal_interval_store = make_interval_store()
|
||
vertical_interval_store = make_interval_store()
|
||
yaw_interval_store = make_interval_store()
|
||
yaw_heading_interval_store = make_interval_store()
|
||
yaw_horizontal_interval_store = make_interval_store()
|
||
yaw_compare_lateral_store = defaultdict(make_metric_bucket)
|
||
large_vehicle_compare_bucket = make_metric_bucket()
|
||
large_vehicle_yaw_compare_lateral_store = defaultdict(make_metric_bucket)
|
||
yaw_compare_lateral_store_by_face_visibility = {
|
||
bucket_name: defaultdict(make_metric_bucket) for bucket_name in FACE_VISIBILITY_BUCKET_ORDER
|
||
}
|
||
|
||
rng = random.Random(int(badcase_random_seed) + (0 if roi_name == "roi0" else 1000))
|
||
bad_yaw_store = make_reservoir_store()
|
||
bad_x_store = make_reservoir_store()
|
||
bad_y_store = make_reservoir_store()
|
||
bad_face_selection_store = make_reservoir_store()
|
||
yaw_visual_stores = make_interval_visual_reservoirs()
|
||
x_visual_stores = make_interval_visual_reservoirs()
|
||
z_visual_stores = make_interval_visual_reservoirs()
|
||
class_badcase_stores = {
|
||
"yaw": defaultdict(make_reservoir_store),
|
||
"horizontal": defaultdict(make_reservoir_store),
|
||
"vertical": defaultdict(make_reservoir_store),
|
||
}
|
||
error_frame_records = {
|
||
"yaw": defaultdict(list),
|
||
"horizontal": defaultdict(list),
|
||
"vertical": defaultdict(list),
|
||
}
|
||
error_bin_per_bucket_capacity = max(1, int(error_bin_badcases)) if error_bin_badcases > 0 else 0
|
||
|
||
total_samples = len(entries)
|
||
start_time = time.time()
|
||
for batch_item, raw_outputs in iter_roi_analysis_samples(
|
||
bundle=bundle,
|
||
entries=entries,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
classes=classes,
|
||
min_wh_px=min_wh_px,
|
||
inference_batch_size=inference_batch_size,
|
||
):
|
||
sample_index = int(batch_item["sample_index"])
|
||
image_path = batch_item["image_path"]
|
||
label_path = batch_item["label_path"]
|
||
prepared = batch_item["prepared"]
|
||
gt = batch_item["gt"]
|
||
|
||
analysis_outputs = filter_prediction_outputs(
|
||
raw_outputs=raw_outputs,
|
||
conf_thres=float(report_confidence_2d),
|
||
max_det=bundle.spec.max_det,
|
||
classes=classes,
|
||
)
|
||
analysis_detections = analysis_outputs.detections
|
||
analysis_preds_3d = analysis_outputs.preds_3d
|
||
analysis_anchors = analysis_outputs.anchors
|
||
analysis_strides = analysis_outputs.strides
|
||
predictions = decode_prepared_roi_predictions(
|
||
bundle=bundle,
|
||
prepared=prepared,
|
||
filtered_outputs=analysis_outputs,
|
||
edge_yaw_max_lateral_dist_m=edge_yaw_max_lateral_dist_m,
|
||
)
|
||
gt_class_names = list(gt.get("class_names") or [])
|
||
|
||
for gt_count_index, cls_id in enumerate(gt["classes"].tolist()):
|
||
detailed_count_name = (
|
||
str(gt_class_names[int(gt_count_index)])
|
||
if int(gt_count_index) < len(gt_class_names)
|
||
else get_cls_name(bundle.names, int(cls_id))
|
||
)
|
||
add_gt_count(overall_bucket)
|
||
add_gt_count(class_buckets[int(cls_id)])
|
||
add_gt_count(detailed_class_buckets[(int(cls_id), detailed_count_name)])
|
||
for prediction in predictions:
|
||
add_prediction_count(overall_bucket)
|
||
add_prediction_count(class_buckets[int(prediction["cls_id"])])
|
||
|
||
img_h, img_w = prepared.image.shape[:2]
|
||
gt_labels_3d = gt["lb_3d"]
|
||
pred_boxes = np.asarray([prediction["bbox_xyxy"] for prediction in predictions], dtype=np.float32) if predictions else np.zeros((0, 4), dtype=np.float32)
|
||
pred_cls = np.asarray([prediction["cls_id"] for prediction in predictions], dtype=np.int32) if predictions else np.zeros((0,), dtype=np.int32)
|
||
pred_conf = np.asarray([prediction["confidence"] for prediction in predictions], dtype=np.float32) if predictions else np.zeros((0,), dtype=np.float32)
|
||
analysis_pred_boxes = (
|
||
np.asarray(analysis_detections[:, :4], dtype=np.float32) if len(analysis_detections) else np.zeros((0, 4), dtype=np.float32)
|
||
)
|
||
analysis_pred_cls = (
|
||
np.asarray(analysis_detections[:, 5], dtype=np.int32).reshape(-1) if len(analysis_detections) else np.zeros((0,), dtype=np.int32)
|
||
)
|
||
analysis_pred_conf = (
|
||
np.asarray(analysis_detections[:, 4], dtype=np.float32).reshape(-1) if len(analysis_detections) else np.zeros((0,), dtype=np.float32)
|
||
)
|
||
gt_focus_mask = np.zeros((len(gt["classes"]),), dtype=bool)
|
||
gt_difficulties = np.asarray(gt["lb_2d"].get("difficulties", []), dtype=np.float32).reshape(-1)
|
||
if gt_labels_3d is not None and len(gt_labels_3d):
|
||
for gt_index, cls_id in enumerate(gt["classes"].tolist()):
|
||
if gt_index >= len(gt_labels_3d):
|
||
break
|
||
gt_attrs_focus = extract_3d_attrs_from_gt(
|
||
gt_labels_3d[gt_index],
|
||
int(cls_id),
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
score_thr=face_visibility_score_thresh,
|
||
)
|
||
if gt_attrs_focus is None:
|
||
continue
|
||
center = np.asarray(gt_attrs_focus.get("center"), dtype=np.float32).reshape(-1)
|
||
lateral_m = to_float(center[0]) if center.size > 0 else None
|
||
longitudinal_m = to_float(center[2]) if center.size > 2 else None
|
||
difficulty = gt_difficulties[gt_index] if gt_index < len(gt_difficulties) else None
|
||
gt_focus_mask[gt_index] = is_focused_confusion_gt_object(
|
||
lateral_m=lateral_m,
|
||
longitudinal_m=longitudinal_m,
|
||
difficulty=difficulty,
|
||
max_abs_longitudinal_m=focused_confusion_max_abs_longitudinal_m,
|
||
)
|
||
analysis_pred_focus_mask = build_prediction_focus_mask(
|
||
analysis_detections,
|
||
analysis_preds_3d,
|
||
analysis_anchors,
|
||
analysis_strides,
|
||
prepared.calib,
|
||
max_abs_longitudinal_m=focused_confusion_max_abs_longitudinal_m,
|
||
)
|
||
two_d_eval_packets.append(
|
||
{
|
||
"sample_index": int(sample_index),
|
||
"image_path": str(image_path),
|
||
"label_path": str(label_path),
|
||
"gt_cls": gt["classes"].copy(),
|
||
"gt_boxes": gt["boxes_xyxy"].copy(),
|
||
"gt_focus_mask": gt_focus_mask.copy(),
|
||
"pred_cls": analysis_pred_cls.copy(),
|
||
"pred_boxes": analysis_pred_boxes.copy(),
|
||
"pred_conf": analysis_pred_conf.copy(),
|
||
"pred_focus_mask": analysis_pred_focus_mask.copy(),
|
||
}
|
||
)
|
||
matches, iou_matrix = greedy_match_indices(gt["classes"], gt["boxes_xyxy"], pred_cls, pred_boxes, iou_thr=0.5)
|
||
face_selection_matches, face_selection_iou_matrix = greedy_match_indices_any_class(gt["boxes_xyxy"], pred_boxes, iou_thr=0.5)
|
||
occlusion_matches, _ = greedy_match_indices_any_class(gt["boxes_xyxy"], pred_boxes, iou_thr=0.5)
|
||
for gt_occ_index, pred_occ_index in occlusion_matches:
|
||
gt_occ_cls_id = int(gt["classes"][gt_occ_index])
|
||
pred_occ_cls_id = int(pred_cls[pred_occ_index])
|
||
add_label_accuracy_sample(
|
||
occlusion_binary_store,
|
||
occlusion_binary_label(get_cls_name(bundle.names, gt_occ_cls_id)),
|
||
occlusion_binary_label(get_cls_name(bundle.names, pred_occ_cls_id)),
|
||
)
|
||
|
||
for gt_face_index, pred_face_index in face_selection_matches:
|
||
if gt_labels_3d is None or gt_face_index >= len(gt_labels_3d):
|
||
continue
|
||
gt_face_cls_id = int(gt["classes"][gt_face_index])
|
||
pred_face_cls_id = int(pred_cls[pred_face_index])
|
||
if gt_face_cls_id not in face_3d_classes:
|
||
continue
|
||
gt_face_decoded = decode_3d_target(
|
||
gt_labels_3d[gt_face_index],
|
||
gt_face_cls_id,
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
score_thr=face_visibility_score_thresh,
|
||
bbox_xyxy=gt["boxes_xyxy"][gt_face_index],
|
||
)
|
||
pred_face_decoded = predictions[pred_face_index].get("decoded")
|
||
if gt_face_decoded is None or pred_face_decoded is None:
|
||
continue
|
||
gt_selection_label = gt_face_selection_label(gt_labels_3d[gt_face_index], gt_face_decoded)
|
||
pred_selection_label = pred_face_selection_label(predictions[pred_face_index]["pred_41"], pred_face_decoded)
|
||
add_label_accuracy_sample(face_selection_store, gt_selection_label, pred_selection_label)
|
||
if gt_selection_label is not None and pred_selection_label is not None and gt_selection_label != pred_selection_label:
|
||
gt_face_cls_name = (
|
||
str(gt_class_names[int(gt_face_index)])
|
||
if int(gt_face_index) < len(gt_class_names)
|
||
else get_cls_name(bundle.names, gt_face_cls_id)
|
||
)
|
||
face_selection_record = make_face_selection_badcase_record(
|
||
sample_index=sample_index,
|
||
roi_name=roi_name,
|
||
image_path=image_path,
|
||
label_path=label_path,
|
||
gt_index=int(gt_face_index),
|
||
pred_index=int(pred_face_index),
|
||
gt_cls_id=gt_face_cls_id,
|
||
gt_cls_name=gt_face_cls_name,
|
||
pred_cls_id=pred_face_cls_id,
|
||
pred_cls_name=get_cls_name(bundle.names, pred_face_cls_id),
|
||
gt_face_selection_label=gt_selection_label,
|
||
pred_face_selection_label=pred_selection_label,
|
||
gt_bbox_xyxy=gt["boxes_xyxy"][gt_face_index],
|
||
pred_bbox_xyxy=pred_boxes[pred_face_index],
|
||
match_iou=float(face_selection_iou_matrix[int(gt_face_index), int(pred_face_index)]) if face_selection_iou_matrix.size else 0.0,
|
||
confidence=float(pred_conf[pred_face_index]),
|
||
)
|
||
bad_face_selection_writer.writerow(face_selection_record_to_csv_row(face_selection_record))
|
||
reservoir_add(bad_face_selection_store, face_selection_record, topk_badcases, rng)
|
||
|
||
for gt_index, pred_index in matches:
|
||
cls_id = int(gt["classes"][gt_index])
|
||
detailed_cls_name = (
|
||
str(gt_class_names[int(gt_index)])
|
||
if int(gt_index) < len(gt_class_names)
|
||
else get_cls_name(bundle.names, cls_id)
|
||
)
|
||
detailed_cls_key = (cls_id, detailed_cls_name)
|
||
cls_bucket = class_buckets[cls_id]
|
||
detailed_cls_bucket = detailed_class_buckets[detailed_cls_key]
|
||
add_2d_match(overall_bucket)
|
||
add_2d_match(cls_bucket)
|
||
|
||
if gt_labels_3d is None or gt_index >= len(gt_labels_3d):
|
||
continue
|
||
gt_visible_faces = (
|
||
select_gt_visible_faces(gt_labels_3d[gt_index], score_thr=face_visibility_score_thresh) if cls_id in face_3d_classes else []
|
||
)
|
||
gt_decoded = decode_3d_target(
|
||
gt_labels_3d[gt_index],
|
||
cls_id,
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
score_thr=face_visibility_score_thresh,
|
||
bbox_xyxy=gt["boxes_xyxy"][gt_index],
|
||
)
|
||
pred_decoded = predictions[pred_index].get("decoded")
|
||
pred_box_attrs = extract_3d_attrs_from_prediction(
|
||
predictions[pred_index]["pred_41"],
|
||
predictions[pred_index]["anchor_xy"],
|
||
float(predictions[pred_index]["stride"]),
|
||
prepared.calib,
|
||
face_type=None,
|
||
pred_edge_60=predictions[pred_index].get("pred_edge_60"),
|
||
)
|
||
pred_position_attrs = predictions[pred_index]["attrs"]
|
||
if gt_decoded is None or pred_box_attrs is None or pred_position_attrs is None or pred_decoded is None:
|
||
continue
|
||
|
||
gt_attrs = extract_3d_attrs_from_gt(
|
||
gt_labels_3d[gt_index],
|
||
cls_id,
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
score_thr=face_visibility_score_thresh,
|
||
)
|
||
if gt_attrs is None:
|
||
continue
|
||
gt_position_attrs = gt_attrs
|
||
position_error_basis = "box_center"
|
||
if cls_id in face_3d_classes:
|
||
eval_face_type = resolve_face_center_eval_face_type(gt_visible_faces, gt_decoded, pred_decoded)
|
||
if eval_face_type is not None:
|
||
gt_face_position_attrs = extract_3d_attrs_from_gt(
|
||
gt_labels_3d[gt_index],
|
||
cls_id,
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
face_type=int(eval_face_type),
|
||
score_thr=face_visibility_score_thresh,
|
||
)
|
||
pred_face_position_attrs = extract_3d_attrs_from_prediction(
|
||
predictions[pred_index]["pred_41"],
|
||
predictions[pred_index]["anchor_xy"],
|
||
float(predictions[pred_index]["stride"]),
|
||
prepared.calib,
|
||
face_type=int(eval_face_type),
|
||
pred_edge_60=predictions[pred_index].get("pred_edge_60"),
|
||
)
|
||
if gt_face_position_attrs is not None and pred_face_position_attrs is not None:
|
||
gt_position_attrs = gt_face_position_attrs
|
||
pred_position_attrs = pred_face_position_attrs
|
||
position_error_basis = "face_center"
|
||
|
||
gt_edge_yaw_faces = (
|
||
select_gt_visible_faces(gt_labels_3d[gt_index], score_thr=edge_yaw_compare_score_thresh)
|
||
if cls_id in face_3d_classes
|
||
else []
|
||
)
|
||
fallback_face_type = max(gt_edge_yaw_faces, key=lambda item: float(item[1][6]))[0] if gt_edge_yaw_faces else None
|
||
gt_visible_yaw = None
|
||
pred_edge_visible_yaw = None
|
||
pred_edge_bucket_visible_yaw = None
|
||
edge_selection = predictions[pred_index].get("edge_selection")
|
||
if fallback_face_type is not None:
|
||
gt_visible_yaw = decode_multi_visible_face_yaw_from_gt(
|
||
gt_labels_3d[gt_index],
|
||
cls_id,
|
||
prepared.calib,
|
||
img_w,
|
||
img_h,
|
||
face_3d_classes,
|
||
complete_3d_classes,
|
||
fallback_face_type=fallback_face_type,
|
||
score_thr=edge_yaw_compare_score_thresh,
|
||
bbox_xyxy=gt["boxes_xyxy"][gt_index],
|
||
)
|
||
edge_selection_yaw = to_float(predictions[pred_index].get("edge_yaw"))
|
||
pred_edge_bucket_visible_yaw = edge_selection_yaw
|
||
pred_edge_visible_yaw = edge_selection_yaw if bool(predictions[pred_index].get("edge_confident")) else None
|
||
|
||
cut_object = bool(is_gt_cut_object(gt_labels_3d[gt_index])) if cls_id in face_3d_classes else False
|
||
position_eligible = bool(cls_id in complete_3d_classes or not cut_object)
|
||
cls_name = detailed_cls_name
|
||
record = make_record(
|
||
sample_index=sample_index,
|
||
roi_name=roi_name,
|
||
image_path=image_path,
|
||
label_path=label_path,
|
||
cls_name=cls_name,
|
||
gt_index=int(gt_index),
|
||
pred_index=int(pred_index),
|
||
gt_box=gt["boxes_xyxy"][gt_index],
|
||
pred_box=predictions[pred_index]["bbox_xyxy"],
|
||
match_iou=float(iou_matrix[gt_index, pred_index]),
|
||
prediction=predictions[pred_index],
|
||
gt_attrs={**gt_attrs, "corners_3d": gt_decoded["corners_3d"]},
|
||
pred_attrs={**pred_box_attrs, "corners_3d": pred_decoded["corners_3d"]},
|
||
gt_position_attrs=None if gt_position_attrs is None else {**gt_position_attrs, "corners_3d": gt_decoded["corners_3d"]},
|
||
pred_position_attrs=None
|
||
if pred_position_attrs is None
|
||
else {**pred_position_attrs, "corners_3d": pred_decoded["corners_3d"]},
|
||
gt_decoded=gt_decoded,
|
||
pred_decoded=pred_decoded,
|
||
gt_visible_faces=gt_visible_faces,
|
||
gt_visible_yaw=to_float(gt_visible_yaw),
|
||
pred_edge_visible_yaw=to_float(pred_edge_visible_yaw),
|
||
pred_edge_bucket_visible_yaw=to_float(pred_edge_bucket_visible_yaw),
|
||
pred_edge_decoded=predictions[pred_index].get("edge_heading_decoded"),
|
||
pred_edge_box=predictions[pred_index].get("edge_box"),
|
||
is_cut_object_flag=cut_object,
|
||
position_eligible=position_eligible,
|
||
position_error_basis=position_error_basis,
|
||
yaw_compare_max_lateral_dist_m=yaw_compare_max_lateral_dist_m,
|
||
pred_yaw_compare_face_types=(() if edge_selection is None else edge_selection.get("face_types", ())),
|
||
pred_yaw_compare_valid=bool(predictions[pred_index].get("edge_confident")),
|
||
)
|
||
add_3d_record(overall_bucket, record, yaw_bad_threshold_deg, horizontal_bad_threshold_m, vertical_bad_threshold_m)
|
||
add_3d_record(cls_bucket, record, yaw_bad_threshold_deg, horizontal_bad_threshold_m, vertical_bad_threshold_m)
|
||
add_3d_record(detailed_cls_bucket, record, yaw_bad_threshold_deg, horizontal_bad_threshold_m, vertical_bad_threshold_m)
|
||
add_lateral_interval_sample(
|
||
horizontal_interval_store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
lateral_m=record.get("position_gt_x_m"),
|
||
value=record["x_abs_m"] if record["position_eligible"] else None,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
lateral_limit_m=HORIZONTAL_LATERAL_RANGE_M,
|
||
)
|
||
add_interval_sample(
|
||
vertical_interval_store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
depth_m=record.get("position_gt_z_m"),
|
||
value=record["z_abs_m"] if record["position_eligible"] else None,
|
||
bin_width_m=VERTICAL_DEPTH_BIN_M,
|
||
)
|
||
add_interval_sample(
|
||
yaw_interval_store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
depth_m=record["gt_depth_m"],
|
||
value=record["yaw_abs_deg"],
|
||
bin_width_m=YAW_DEPTH_BIN_M,
|
||
)
|
||
add_heading_interval_sample(
|
||
yaw_heading_interval_store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
heading_deg=record["gt_yaw_deg"],
|
||
value=record["yaw_abs_deg"],
|
||
bin_width_deg=YAW_HEADING_BIN_DEG,
|
||
relative_reference_value=(
|
||
None if record["gt_yaw_deg"] is None else wrap_heading_deg(float(record["gt_yaw_deg"]))
|
||
),
|
||
relative_reference_abs=True,
|
||
)
|
||
add_lateral_interval_sample(
|
||
yaw_horizontal_interval_store,
|
||
cls_id=cls_id,
|
||
cls_name=cls_name,
|
||
lateral_m=record["gt_x_m"],
|
||
value=record["yaw_abs_deg"],
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
lateral_limit_m=HORIZONTAL_LATERAL_RANGE_M,
|
||
)
|
||
yaw_compare_lateral_start = lateral_interval_start(
|
||
record["gt_x_m"],
|
||
HORIZONTAL_LATERAL_BIN_M,
|
||
yaw_compare_max_lateral_dist_m,
|
||
)
|
||
face_visibility_bucket = (
|
||
record.get("yaw_compare_face_bucket")
|
||
if cls_id in face_3d_classes
|
||
else None
|
||
)
|
||
large_vehicle_focus_match = bool(cls_id in LARGE_VEHICLE_CLASS_IDS and face_visibility_bucket == "two-face")
|
||
if large_vehicle_focus_match:
|
||
add_3d_record(
|
||
large_vehicle_compare_bucket,
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key="edge_bucket_visible_yaw_abs_deg",
|
||
direct_minus_edge_key="direct_minus_edge_bucket_visible_yaw_abs_deg",
|
||
)
|
||
if yaw_compare_lateral_start is not None and is_signed_lateral_yaw_compare_longitudinal_eligible(record):
|
||
add_3d_record(
|
||
yaw_compare_lateral_store[float(yaw_compare_lateral_start)],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
)
|
||
if face_visibility_bucket in yaw_compare_lateral_store_by_face_visibility:
|
||
add_3d_record(
|
||
yaw_compare_lateral_store_by_face_visibility[str(face_visibility_bucket)][float(yaw_compare_lateral_start)],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key="edge_bucket_visible_yaw_abs_deg",
|
||
direct_minus_edge_key="direct_minus_edge_bucket_visible_yaw_abs_deg",
|
||
)
|
||
if large_vehicle_focus_match:
|
||
add_3d_record(
|
||
large_vehicle_yaw_compare_lateral_store[float(yaw_compare_lateral_start)],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key="edge_bucket_visible_yaw_abs_deg",
|
||
direct_minus_edge_key="direct_minus_edge_bucket_visible_yaw_abs_deg",
|
||
)
|
||
|
||
breakdown_keys = {
|
||
"cut_status": "cut" if cut_object else ("non_cut" if cls_id in face_3d_classes else "n/a"),
|
||
"distance_bin": record["distance_bin"],
|
||
"bbox_diag_bin": record["bbox_diag_bin"],
|
||
"class_group": "face_3d" if cls_id in face_3d_classes else ("complete_3d" if cls_id in complete_3d_classes else "2d_only"),
|
||
}
|
||
if face_visibility_bucket is not None:
|
||
breakdown_keys["face_visibility"] = face_visibility_bucket
|
||
add_3d_record(
|
||
class_face_visibility_buckets[cls_id][face_visibility_bucket],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key="edge_bucket_visible_yaw_abs_deg",
|
||
direct_minus_edge_key="direct_minus_edge_bucket_visible_yaw_abs_deg",
|
||
)
|
||
add_3d_record(
|
||
detailed_class_face_visibility_buckets[detailed_cls_key][face_visibility_bucket],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key="edge_bucket_visible_yaw_abs_deg",
|
||
direct_minus_edge_key="direct_minus_edge_bucket_visible_yaw_abs_deg",
|
||
)
|
||
for breakdown_name, breakdown_key in breakdown_keys.items():
|
||
add_3d_record(
|
||
breakdowns[breakdown_name][breakdown_key],
|
||
record,
|
||
yaw_bad_threshold_deg,
|
||
horizontal_bad_threshold_m,
|
||
vertical_bad_threshold_m,
|
||
edge_visible_key=(
|
||
"edge_bucket_visible_yaw_abs_deg" if breakdown_name == "face_visibility" else "edge_visible_yaw_abs_deg"
|
||
),
|
||
direct_minus_edge_key=(
|
||
"direct_minus_edge_bucket_visible_yaw_abs_deg"
|
||
if breakdown_name == "face_visibility"
|
||
else "direct_minus_edge_visible_yaw_abs_deg"
|
||
),
|
||
)
|
||
|
||
if record["yaw_abs_deg"] is not None and record["yaw_abs_deg"] >= yaw_bad_threshold_deg:
|
||
bad_yaw_writer.writerow(record_to_csv_row(record))
|
||
reservoir_add(bad_yaw_store, record, topk_badcases, rng)
|
||
reservoir_add(class_badcase_stores["yaw"][(cls_id, cls_name)], record, per_class_badcases, rng)
|
||
error_frame_records["yaw"][str(record["image_path"])].append(dict(record))
|
||
add_interval_visual_record(
|
||
yaw_visual_stores,
|
||
record["yaw_abs_deg"],
|
||
record,
|
||
bin_width=ERROR_YAW_BIN_DEG,
|
||
per_bin_capacity=error_bin_per_bucket_capacity,
|
||
rng=rng,
|
||
)
|
||
|
||
if record["position_eligible"] and record["x_abs_m"] is not None and record["x_abs_m"] > horizontal_bad_threshold_m:
|
||
bad_x_writer.writerow(record_to_csv_row(record))
|
||
reservoir_add(bad_x_store, record, topk_badcases, rng)
|
||
reservoir_add(class_badcase_stores["horizontal"][(cls_id, cls_name)], record, per_class_badcases, rng)
|
||
error_frame_records["horizontal"][str(record["image_path"])].append(dict(record))
|
||
add_interval_visual_record(
|
||
x_visual_stores,
|
||
record["x_abs_m"],
|
||
record,
|
||
bin_width=ERROR_DISTANCE_BIN_M,
|
||
per_bin_capacity=error_bin_per_bucket_capacity,
|
||
rng=rng,
|
||
)
|
||
|
||
if record["position_eligible"] and record["z_abs_m"] is not None and record["z_abs_m"] > vertical_bad_threshold_m:
|
||
bad_y_writer.writerow(record_to_csv_row(record))
|
||
reservoir_add(bad_y_store, record, topk_badcases, rng)
|
||
reservoir_add(class_badcase_stores["vertical"][(cls_id, cls_name)], record, per_class_badcases, rng)
|
||
error_frame_records["vertical"][str(record["image_path"])].append(dict(record))
|
||
add_interval_visual_record(
|
||
z_visual_stores,
|
||
record["z_abs_m"],
|
||
record,
|
||
bin_width=ERROR_DISTANCE_BIN_M,
|
||
per_bin_capacity=error_bin_per_bucket_capacity,
|
||
rng=rng,
|
||
)
|
||
|
||
if (sample_index + 1) % max(1, log_every) == 0 or (sample_index + 1) == total_samples:
|
||
elapsed = time.time() - start_time
|
||
per_sample = elapsed / max(sample_index + 1, 1)
|
||
remaining = total_samples - (sample_index + 1)
|
||
eta = remaining * per_sample
|
||
print(
|
||
f"[{roi_name}] {sample_index + 1}/{total_samples} "
|
||
f"elapsed={elapsed / 60:.1f}m eta={eta / 60:.1f}m matched_3d={overall_bucket['matched_3d']}"
|
||
)
|
||
|
||
bad_yaw_handle.close()
|
||
bad_x_handle.close()
|
||
bad_y_handle.close()
|
||
bad_face_selection_handle.close()
|
||
|
||
thresholded_2d = build_thresholded_2d_artifacts(
|
||
eval_packets=two_d_eval_packets,
|
||
roi_name=roi_name,
|
||
names_dict=names_dict,
|
||
confidence_threshold=float(report_confidence_2d),
|
||
topk_badcases=int(topk_badcases),
|
||
badcase_random_seed=int(badcase_random_seed),
|
||
roi_output=roi_output,
|
||
bundle=bundle,
|
||
image_root=image_root,
|
||
max_abs_longitudinal_m=focused_confusion_max_abs_longitudinal_m,
|
||
)
|
||
classification_summary_2d = thresholded_2d["classification_summary_2d"]
|
||
focused_classification_summary_2d = thresholded_2d["focused_classification_summary_2d"]
|
||
face_selection_summary = summarize_label_accuracy_store(face_selection_store)
|
||
occlusion_binary_summary = summarize_label_accuracy_store(occlusion_binary_store)
|
||
|
||
overall_summary = summarize_metric_bucket(overall_bucket)
|
||
overall_summary.update(thresholded_2d["overall"])
|
||
overall_summary["map50_2d"] = ap_summary_2d.get("map50")
|
||
overall_summary["map50_95_2d"] = ap_summary_2d.get("map50_95")
|
||
overall_summary["precision_ap_2d"] = threshold_advice_2d.get("mean_precision")
|
||
overall_summary["recall_ap_2d"] = threshold_advice_2d.get("mean_recall")
|
||
overall_summary["f1_ap_2d"] = threshold_advice_2d.get("mean_f1")
|
||
overall_summary["recommended_confidence_2d"] = float(report_confidence_2d)
|
||
overall_summary["report_confidence_2d"] = float(report_confidence_2d)
|
||
overall_summary["cls_acc_2d"] = classification_summary_2d.get("overall_accuracy")
|
||
overall_summary["cls_eval_pairs_2d"] = int(classification_summary_2d.get("overall_pairs") or 0)
|
||
total_yaw_sum = overall_summary["yaw_abs_sum"] or 0.0
|
||
|
||
class_rows = []
|
||
detailed_class_keys = sorted(detailed_class_buckets.keys(), key=lambda item: (int(item[0]), str(item[1])))
|
||
class_ids_with_2d_only = sorted(set((thresholded_2d.get("per_class") or {}).keys()) - {int(key[0]) for key in detailed_class_keys})
|
||
class_keys = [*detailed_class_keys, *[(int(cls_id), get_cls_name(bundle.names, int(cls_id))) for cls_id in class_ids_with_2d_only]]
|
||
for cls_id, detailed_cls_name in class_keys:
|
||
bucket = detailed_class_buckets.get((int(cls_id), str(detailed_cls_name)), make_metric_bucket())
|
||
row = summarize_metric_bucket(bucket)
|
||
two_face_row = summarize_metric_bucket(
|
||
(detailed_class_face_visibility_buckets.get((int(cls_id), str(detailed_cls_name)), {}) or {}).get("two-face", make_metric_bucket())
|
||
)
|
||
row["cls_id"] = int(cls_id)
|
||
row["cls_name"] = str(detailed_cls_name)
|
||
row["mapped_cls_name"] = get_cls_name(bundle.names, int(cls_id))
|
||
row["yaw_contribution_rate"] = (row["yaw_abs_sum"] / total_yaw_sum) if total_yaw_sum > 0 else None
|
||
thresholded_2d_row = (thresholded_2d.get("per_class") or {}).get(int(cls_id), {})
|
||
row["gt_total"] = int(thresholded_2d_row.get("gt_total", row.get("gt_total", 0)) or 0)
|
||
row["pred_total"] = int(thresholded_2d_row.get("pred_total", row.get("pred_total", 0)) or 0)
|
||
row["matched_2d"] = int(thresholded_2d_row.get("matched_2d", row.get("matched_2d", 0)) or 0)
|
||
row["precision_2d"] = thresholded_2d_row.get("precision_2d")
|
||
row["recall_2d"] = thresholded_2d_row.get("recall_2d")
|
||
row["f1_2d"] = thresholded_2d_row.get("f1_2d")
|
||
ap_row = ap_summary_2d["per_class"].get(int(cls_id), {})
|
||
row["map50_2d"] = ap_row.get("map50")
|
||
row["map50_95_2d"] = ap_row.get("map50_95")
|
||
row["precision_ap_2d"] = ap_row.get("precision_ap")
|
||
row["recall_ap_2d"] = ap_row.get("recall_ap")
|
||
row["f1_ap_2d"] = ap_row.get("f1_at_recommended_confidence")
|
||
row["optimal_confidence_2d"] = ap_row.get("optimal_confidence")
|
||
row["optimal_f1_2d"] = ap_row.get("optimal_f1")
|
||
row["false_negatives_2d"] = int(
|
||
thresholded_2d_row.get("false_negatives_2d", max(0, int(row.get("gt_total", 0)) - int(row.get("matched_2d", 0))))
|
||
)
|
||
row["false_positives_2d"] = int(
|
||
thresholded_2d_row.get("false_positives_2d", max(0, int(row.get("pred_total", 0)) - int(row.get("matched_2d", 0))))
|
||
)
|
||
cls_classification = classification_summary_2d["per_class"].get(int(cls_id), {})
|
||
row["cls_eval_pairs_2d"] = int(cls_classification.get("cls_eval_pairs_2d") or 0)
|
||
row["cls_correct_2d"] = int(cls_classification.get("cls_correct_2d") or 0)
|
||
row["cls_acc_2d"] = cls_classification.get("cls_acc_2d")
|
||
row["overall_yaw_compare_count"] = int(row.get("yaw_compare_count") or 0)
|
||
row["two_face_matched_3d"] = int(two_face_row.get("matched_3d") or 0)
|
||
row["two_face_compare_count"] = int(two_face_row.get("yaw_compare_count") or 0)
|
||
row["direct_regression_yaw_mae_deg"] = two_face_row.get("direct_regression_yaw_mae_deg")
|
||
row["direct_regression_yaw_p90_deg"] = two_face_row.get("direct_regression_yaw_p90_deg")
|
||
row["edge_based_yaw_mae_deg"] = two_face_row.get("edge_based_yaw_mae_deg")
|
||
row["edge_based_yaw_p90_deg"] = two_face_row.get("edge_based_yaw_p90_deg")
|
||
row["mean_direct_minus_edge_yaw_deg"] = two_face_row.get("mean_direct_minus_edge_yaw_deg")
|
||
row["median_direct_minus_edge_yaw_deg"] = two_face_row.get("median_direct_minus_edge_yaw_deg")
|
||
row["yaw_compare_edge_better_count"] = int(two_face_row.get("yaw_compare_edge_better_count") or 0)
|
||
row["yaw_compare_direct_better_count"] = int(two_face_row.get("yaw_compare_direct_better_count") or 0)
|
||
row["yaw_compare_tie_count"] = int(two_face_row.get("yaw_compare_tie_count") or 0)
|
||
row["yaw_compare_edge_better_rate"] = two_face_row.get("yaw_compare_edge_better_rate")
|
||
row["yaw_compare_direct_better_rate"] = two_face_row.get("yaw_compare_direct_better_rate")
|
||
row["yaw_compare_tie_rate"] = two_face_row.get("yaw_compare_tie_rate")
|
||
class_rows.append(row)
|
||
|
||
class_rows_sorted = sorted(
|
||
class_rows,
|
||
key=lambda row: ((row.get("yaw_abs_sum") or 0.0), row.get("matched_3d") or 0),
|
||
reverse=True,
|
||
)
|
||
write_class_csv(roi_output / "class_metrics.csv", class_rows)
|
||
horizontal_interval_rows = build_interval_rows(
|
||
horizontal_interval_store,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
value_name="x_abs_m",
|
||
threshold=horizontal_bad_threshold_m,
|
||
relative_value_name="x_abs_pct",
|
||
relative_reference_abs=True,
|
||
)
|
||
vertical_interval_rows = build_interval_rows(
|
||
vertical_interval_store,
|
||
bin_width_m=VERTICAL_DEPTH_BIN_M,
|
||
value_name="z_abs_m",
|
||
threshold=vertical_bad_threshold_m,
|
||
relative_value_name="z_abs_pct",
|
||
)
|
||
if roi_name == "roi1":
|
||
# ROI1 near-range longitudinal z error is not meaningful in the report.
|
||
vertical_interval_rows = [
|
||
row for row in vertical_interval_rows if float(row.get("depth_bin_start_m", 0.0)) >= ROI1_MIN_Z_ERROR_DEPTH_M
|
||
]
|
||
yaw_interval_rows = build_interval_rows(
|
||
yaw_interval_store,
|
||
bin_width_m=YAW_DEPTH_BIN_M,
|
||
value_name="yaw_abs_deg",
|
||
threshold=yaw_bad_threshold_deg,
|
||
)
|
||
yaw_heading_interval_rows = build_interval_rows(
|
||
yaw_heading_interval_store,
|
||
bin_width_m=YAW_HEADING_BIN_DEG,
|
||
value_name="yaw_abs_deg",
|
||
threshold=yaw_bad_threshold_deg,
|
||
relative_value_name="yaw_abs_pct",
|
||
interval_prefix="yaw_bin",
|
||
interval_unit="deg",
|
||
relative_reference_label="mean |gt_yaw|",
|
||
relative_reference_unit="deg",
|
||
relative_reference_field="relative_gt_yaw_deg",
|
||
)
|
||
yaw_horizontal_interval_rows = build_interval_rows(
|
||
yaw_horizontal_interval_store,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
value_name="yaw_abs_deg",
|
||
threshold=yaw_bad_threshold_deg,
|
||
)
|
||
yaw_compare_lateral_rows = build_metric_bucket_interval_rows(
|
||
yaw_compare_lateral_store,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
prefix="lateral_bin",
|
||
)
|
||
yaw_compare_lateral_rows_by_face_visibility = build_grouped_metric_bucket_interval_rows(
|
||
yaw_compare_lateral_store_by_face_visibility,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
prefix="lateral_bin",
|
||
group_order=FACE_VISIBILITY_BUCKET_ORDER,
|
||
)
|
||
large_vehicle_yaw_compare_lateral_rows = build_metric_bucket_interval_rows(
|
||
large_vehicle_yaw_compare_lateral_store,
|
||
bin_width_m=HORIZONTAL_LATERAL_BIN_M,
|
||
prefix="lateral_bin",
|
||
)
|
||
yaw_compare_lateral_rows_by_face_visibility_flat = flatten_grouped_metric_bucket_interval_rows(
|
||
yaw_compare_lateral_rows_by_face_visibility,
|
||
group_key="face_visibility_bucket",
|
||
)
|
||
horizontal_interval_csv = roi_output / "class_horizontal_lateral_5m.csv"
|
||
vertical_interval_csv = roi_output / "class_vertical_depth_5m.csv"
|
||
yaw_interval_csv = roi_output / "class_yaw_depth_10m.csv"
|
||
yaw_heading_interval_csv = roi_output / "class_yaw_heading_10deg.csv"
|
||
yaw_horizontal_interval_csv = roi_output / "class_yaw_horizontal_5m.csv"
|
||
yaw_compare_lateral_csv = roi_output / "yaw_compare_signed_lateral_5m.csv"
|
||
yaw_compare_lateral_by_face_visibility_csv = roi_output / "yaw_compare_signed_lateral_5m_by_face_visibility.csv"
|
||
face_selection_csv = roi_output / "face_selection_accuracy.csv"
|
||
face_selection_confusion_csv = roi_output / "face_selection_confusion_matrix.csv"
|
||
fake_class_csv = roi_output / "fake_class_accuracy.csv"
|
||
occlusion_binary_csv = roi_output / "occlusion_binary_accuracy.csv"
|
||
write_rows_csv(
|
||
horizontal_interval_csv,
|
||
horizontal_interval_rows,
|
||
fallback_fields=["cls_id", "cls_name", "depth_bin_start_m", "depth_bin_end_m", "depth_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
vertical_interval_csv,
|
||
vertical_interval_rows,
|
||
fallback_fields=["cls_id", "cls_name", "depth_bin_start_m", "depth_bin_end_m", "depth_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
yaw_interval_csv,
|
||
yaw_interval_rows,
|
||
fallback_fields=["cls_id", "cls_name", "depth_bin_start_m", "depth_bin_end_m", "depth_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
yaw_heading_interval_csv,
|
||
yaw_heading_interval_rows,
|
||
fallback_fields=["cls_id", "cls_name", "yaw_bin_start_deg", "yaw_bin_end_deg", "yaw_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
yaw_horizontal_interval_csv,
|
||
yaw_horizontal_interval_rows,
|
||
fallback_fields=["cls_id", "cls_name", "depth_bin_start_m", "depth_bin_end_m", "depth_bin_label", "count"],
|
||
)
|
||
write_rows_csv(
|
||
yaw_compare_lateral_csv,
|
||
yaw_compare_lateral_rows,
|
||
fallback_fields=["lateral_bin_start_m", "lateral_bin_end_m", "lateral_bin_label", "yaw_compare_count"],
|
||
)
|
||
write_rows_csv(
|
||
yaw_compare_lateral_by_face_visibility_csv,
|
||
yaw_compare_lateral_rows_by_face_visibility_flat,
|
||
fallback_fields=["face_visibility_bucket", "lateral_bin_start_m", "lateral_bin_end_m", "lateral_bin_label", "yaw_compare_count"],
|
||
)
|
||
write_rows_csv(
|
||
face_selection_csv,
|
||
label_accuracy_rows(face_selection_summary, label_key="selection"),
|
||
fallback_fields=["selection", "total", "correct", "accuracy"],
|
||
)
|
||
write_rows_csv(
|
||
face_selection_confusion_csv,
|
||
label_confusion_matrix_rows(face_selection_summary, label_key="gt_selection"),
|
||
fallback_fields=["gt_selection", *list(face_selection_summary.get("label_order", FACE_SELECTION_LABEL_ORDER))],
|
||
)
|
||
write_rows_csv(
|
||
fake_class_csv,
|
||
label_accuracy_rows(thresholded_2d.get("fake_class_summary", {}), label_key="fake_class"),
|
||
fallback_fields=["fake_class", "total", "correct", "accuracy"],
|
||
)
|
||
write_rows_csv(
|
||
occlusion_binary_csv,
|
||
label_accuracy_rows(occlusion_binary_summary, label_key="occlusion"),
|
||
fallback_fields=["occlusion", "total", "correct", "accuracy"],
|
||
)
|
||
|
||
breakdown_summaries = {
|
||
name: {key: summarize_metric_bucket(bucket) for key, bucket in group.items()} for name, group in breakdowns.items()
|
||
}
|
||
write_json(roi_output / "breakdowns.json", breakdown_summaries)
|
||
yaw_compare_diagnostics = build_yaw_compare_diagnostics(overall_summary, breakdown_summaries)
|
||
large_vehicle_compare_summary = summarize_metric_bucket(large_vehicle_compare_bucket)
|
||
large_vehicle_class_rows = sorted(
|
||
[row for row in class_rows if int(row.get("cls_id", -1)) in LARGE_VEHICLE_CLASS_IDS],
|
||
key=lambda row: (
|
||
int(row.get("two_face_compare_count") or 0),
|
||
int(row.get("matched_3d") or 0),
|
||
float(row.get("yaw_abs_sum") or 0.0),
|
||
),
|
||
reverse=True,
|
||
)
|
||
|
||
bad_yaw_records = reservoir_records(bad_yaw_store, rng)
|
||
bad_x_records = reservoir_records(bad_x_store, rng)
|
||
bad_y_records = reservoir_records(bad_y_store, rng)
|
||
bad_face_selection_records = reservoir_records(bad_face_selection_store, rng)
|
||
yaw_error_visual_records = {
|
||
float(start): reservoir_records(store, rng)
|
||
for start, store in sorted(yaw_visual_stores.items(), key=lambda item: item[0], reverse=True)
|
||
if store.get("records")
|
||
}
|
||
horizontal_error_visual_records = {
|
||
float(start): reservoir_records(store, rng)
|
||
for start, store in sorted(x_visual_stores.items(), key=lambda item: item[0], reverse=True)
|
||
if store.get("records")
|
||
}
|
||
vertical_error_visual_records = {
|
||
float(start): reservoir_records(store, rng)
|
||
for start, store in sorted(z_visual_stores.items(), key=lambda item: item[0], reverse=True)
|
||
if store.get("records")
|
||
}
|
||
error_bin_badcase_manifests = {
|
||
"yaw": save_interval_badcase_visuals(
|
||
yaw_error_visual_records,
|
||
error_frame_records["yaw"],
|
||
roi_output / "visuals" / "yaw_bins",
|
||
"yaw",
|
||
ERROR_YAW_BIN_DEG,
|
||
"deg",
|
||
max(1, int(error_bin_samples_per_bin)),
|
||
bundle,
|
||
image_root,
|
||
),
|
||
"horizontal": save_interval_badcase_visuals(
|
||
horizontal_error_visual_records,
|
||
error_frame_records["horizontal"],
|
||
roi_output / "visuals" / "horizontal_bins",
|
||
"horizontal",
|
||
ERROR_DISTANCE_BIN_M,
|
||
"m",
|
||
max(1, int(error_bin_samples_per_bin)),
|
||
bundle,
|
||
image_root,
|
||
),
|
||
"vertical": save_interval_badcase_visuals(
|
||
vertical_error_visual_records,
|
||
error_frame_records["vertical"],
|
||
roi_output / "visuals" / "vertical_bins",
|
||
"vertical",
|
||
ERROR_DISTANCE_BIN_M,
|
||
"m",
|
||
max(1, int(error_bin_samples_per_bin)),
|
||
bundle,
|
||
image_root,
|
||
),
|
||
}
|
||
class_badcase_manifests = {
|
||
category: save_class_badcase_visuals(
|
||
{
|
||
class_key: reservoir_records(store, rng)
|
||
for class_key, store in stores.items()
|
||
},
|
||
roi_output / "visuals" / f"{category}_by_class",
|
||
category,
|
||
bundle,
|
||
image_root,
|
||
)
|
||
for category, stores in class_badcase_stores.items()
|
||
}
|
||
face_selection_manifest = save_face_selection_badcase_visuals(
|
||
bad_face_selection_records,
|
||
roi_output / "visuals" / "face_selection",
|
||
"face_selection",
|
||
bundle,
|
||
image_root,
|
||
)
|
||
|
||
worst_depth_bins = sort_nullable_desc(breakdown_summaries["distance_bin"].items(), "yaw_mae_deg")[:5]
|
||
worst_bbox_bins = sort_nullable_desc(breakdown_summaries["bbox_diag_bin"].items(), "yaw_mae_deg")[:5]
|
||
worst_face_buckets = sort_nullable_desc(breakdown_summaries["face_visibility"].items(), "yaw_mae_deg")[:5]
|
||
|
||
top_horizontal_classes = sorted(class_rows, key=lambda row: row.get("x_bad_count") or 0, reverse=True)[:10]
|
||
top_vertical_classes = sorted(class_rows, key=lambda row: row.get("z_bad_count") or 0, reverse=True)[:10]
|
||
top_yaw_contributors = class_rows_sorted[:10]
|
||
horizontal_by_class = group_rows_by_class(horizontal_interval_rows)
|
||
vertical_by_class = group_rows_by_class(vertical_interval_rows)
|
||
yaw_by_class = group_rows_by_class(yaw_interval_rows)
|
||
per_class_interval_insights = []
|
||
for row in class_rows_sorted:
|
||
if int(row.get("matched_3d") or 0) <= 0:
|
||
continue
|
||
cls_key = (int(row["cls_id"]), str(row["cls_name"]))
|
||
horizontal_bins = top_interval_rows(horizontal_by_class.get(cls_key, []), "mean_x_abs_m", topn=3, min_count=30)
|
||
vertical_bins = top_interval_rows(vertical_by_class.get(cls_key, []), "mean_z_abs_m", topn=3, min_count=30)
|
||
yaw_bins = top_interval_rows(yaw_by_class.get(cls_key, []), "mean_yaw_abs_deg", topn=3, min_count=30)
|
||
per_class_insight = {
|
||
"cls_id": int(row["cls_id"]),
|
||
"cls_name": str(row["cls_name"]),
|
||
"matched_3d": int(row.get("matched_3d") or 0),
|
||
"matched_pos": int(row.get("matched_pos") or 0),
|
||
"yaw_mae_deg": row.get("yaw_mae_deg"),
|
||
"x_abs_mae_m": row.get("x_abs_mae_m"),
|
||
"y_abs_mae_m": row.get("y_abs_mae_m"),
|
||
"z_abs_mae_m": row.get("z_abs_mae_m"),
|
||
"yaw_bad_rate": row.get("yaw_bad_rate"),
|
||
"x_bad_rate": row.get("x_bad_rate"),
|
||
"y_bad_rate": row.get("y_bad_rate"),
|
||
"z_bad_rate": row.get("z_bad_rate"),
|
||
"horizontal_bins": horizontal_bins,
|
||
"vertical_bins": vertical_bins,
|
||
"yaw_bins": yaw_bins,
|
||
"horizontal_bins_text": ";".join(
|
||
f"{bin_row['depth_bin_label']}: mean={format_float(bin_row.get('mean_x_abs_m'))}m, p90={format_float(bin_row.get('p90_x_abs_m'))}m, n={bin_row['count']}"
|
||
for bin_row in horizontal_bins
|
||
)
|
||
or "n/a",
|
||
"vertical_bins_text": ";".join(
|
||
f"{bin_row['depth_bin_label']}: mean={format_float(bin_row.get('mean_z_abs_m'))}m, p90={format_float(bin_row.get('p90_z_abs_m'))}m, n={bin_row['count']}"
|
||
for bin_row in vertical_bins
|
||
)
|
||
or "n/a",
|
||
"yaw_bins_text": ";".join(
|
||
f"{bin_row['depth_bin_label']}: mean={format_float(bin_row.get('mean_yaw_abs_deg'), 2)}deg, p90={format_float(bin_row.get('p90_yaw_abs_deg'), 2)}deg, n={bin_row['count']}"
|
||
for bin_row in yaw_bins
|
||
)
|
||
or "n/a",
|
||
}
|
||
per_class_interval_insights.append(per_class_insight)
|
||
|
||
summary_payload = {
|
||
"roi": roi_name,
|
||
"model_path": bundle.spec.model_path,
|
||
"yaw_compare_max_lateral_dist_m": float(yaw_compare_max_lateral_dist_m),
|
||
"yaw_compare_max_longitudinal_dist_m": float(DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M),
|
||
"min_wh_px": float(min_wh_px),
|
||
"configured_confidence_2d": float(configured_confidence_2d),
|
||
"report_confidence_2d": float(report_confidence_2d),
|
||
"horizontal_bad_threshold_m": float(horizontal_bad_threshold_m),
|
||
"vertical_bad_threshold_m": float(vertical_bad_threshold_m),
|
||
"yaw_bad_threshold_deg": float(yaw_bad_threshold_deg),
|
||
"badcase_random_seed": int(badcase_random_seed),
|
||
"per_class_badcases": int(per_class_badcases),
|
||
"error_bin_badcases": int(error_bin_badcases),
|
||
"error_bin_samples_per_bin": int(error_bin_samples_per_bin),
|
||
"overall": overall_summary,
|
||
"ap_summary_2d": ap_summary_2d,
|
||
"threshold_advice_2d": threshold_advice_2d,
|
||
"confidence_curve_paths_2d": ap_summary_2d.get("curve_paths", {}),
|
||
"classification_summary_2d": classification_summary_2d,
|
||
"face_selection_summary": face_selection_summary,
|
||
"face_selection_csv": str(face_selection_csv),
|
||
"face_selection_confusion_csv": str(face_selection_confusion_csv),
|
||
"fake_class_summary": thresholded_2d.get("fake_class_summary", {}),
|
||
"fake_class_csv": str(fake_class_csv),
|
||
"occlusion_binary_summary": occlusion_binary_summary,
|
||
"occlusion_binary_csv": str(occlusion_binary_csv),
|
||
"confusion_matrix_plot_path_2d": thresholded_2d.get("confusion_matrix_plot_path_2d"),
|
||
"focused_classification_summary_2d": focused_classification_summary_2d,
|
||
"focused_confusion_matrix_plot_path_2d": thresholded_2d.get("focused_confusion_matrix_plot_path_2d"),
|
||
"focused_confusion_filter": thresholded_2d.get("focused_confusion_filter", {}),
|
||
"class_rows": class_rows,
|
||
"horizontal_interval_rows": horizontal_interval_rows,
|
||
"vertical_interval_rows": vertical_interval_rows,
|
||
"yaw_interval_rows": yaw_interval_rows,
|
||
"yaw_heading_interval_rows": yaw_heading_interval_rows,
|
||
"yaw_horizontal_interval_rows": yaw_horizontal_interval_rows,
|
||
"yaw_compare_lateral_rows": yaw_compare_lateral_rows,
|
||
"yaw_compare_lateral_rows_by_face_visibility": yaw_compare_lateral_rows_by_face_visibility,
|
||
"large_vehicle_compare": {
|
||
"class_ids": sorted(int(cls_id) for cls_id in LARGE_VEHICLE_CLASS_IDS),
|
||
"class_scope": LARGE_VEHICLE_CLASS_SCOPE_TEXT,
|
||
"summary": large_vehicle_compare_summary,
|
||
"class_rows": large_vehicle_class_rows,
|
||
"yaw_compare_lateral_rows": large_vehicle_yaw_compare_lateral_rows,
|
||
},
|
||
"horizontal_interval_csv": str(horizontal_interval_csv),
|
||
"vertical_interval_csv": str(vertical_interval_csv),
|
||
"yaw_interval_csv": str(yaw_interval_csv),
|
||
"yaw_heading_interval_csv": str(yaw_heading_interval_csv),
|
||
"yaw_horizontal_interval_csv": str(yaw_horizontal_interval_csv),
|
||
"yaw_compare_lateral_csv": str(yaw_compare_lateral_csv),
|
||
"yaw_compare_lateral_by_face_visibility_csv": str(yaw_compare_lateral_by_face_visibility_csv),
|
||
"per_class_interval_insights": per_class_interval_insights,
|
||
"yaw_compare_diagnostics": yaw_compare_diagnostics,
|
||
"top_yaw_contributors": top_yaw_contributors,
|
||
"top_horizontal_classes": top_horizontal_classes,
|
||
"top_vertical_classes": top_vertical_classes,
|
||
"worst_depth_bins": worst_depth_bins,
|
||
"worst_bbox_bins": worst_bbox_bins,
|
||
"worst_face_buckets": worst_face_buckets,
|
||
"badcase_counts": {
|
||
"yaw_saved_topk": len(bad_yaw_records),
|
||
"horizontal_saved_topk": len(bad_x_records),
|
||
"vertical_saved_topk": len(bad_y_records),
|
||
"face_selection_saved_topk": len(bad_face_selection_records),
|
||
**(thresholded_2d.get("badcase_counts") or {}),
|
||
},
|
||
"badcase_manifest_paths": {
|
||
"face_selection": str(roi_output / "visuals" / "face_selection" / "manifest.json"),
|
||
**(thresholded_2d.get("badcase_manifest_paths") or {}),
|
||
},
|
||
"error_bin_badcase_manifest_entries": error_bin_badcase_manifests,
|
||
"class_badcase_manifest_paths": class_badcase_manifests,
|
||
"badcase_manifest_sizes": {
|
||
**(thresholded_2d.get("badcase_manifest_sizes") or {}),
|
||
"yaw_by_error": sum(int(entry.get("count", 0)) for entry in error_bin_badcase_manifests["yaw"]),
|
||
"horizontal_by_error": sum(int(entry.get("count", 0)) for entry in error_bin_badcase_manifests["horizontal"]),
|
||
"vertical_by_error": sum(int(entry.get("count", 0)) for entry in error_bin_badcase_manifests["vertical"]),
|
||
"face_selection": len(face_selection_manifest),
|
||
},
|
||
}
|
||
write_json(roi_output / "summary.json", summary_payload)
|
||
write_markdown_summary(
|
||
roi_output / "summary.md",
|
||
roi_name=roi_name,
|
||
payload=summary_payload,
|
||
model_path=bundle.spec.model_path,
|
||
split_file=split_file,
|
||
)
|
||
write_markdown_summary_zh(
|
||
roi_output / "summary_zh.md",
|
||
roi_name=roi_name,
|
||
payload=summary_payload,
|
||
model_path=bundle.spec.model_path,
|
||
split_file=split_file,
|
||
horizontal_csv=horizontal_interval_csv,
|
||
vertical_csv=vertical_interval_csv,
|
||
yaw_depth_csv=yaw_interval_csv,
|
||
yaw_heading_csv=yaw_heading_interval_csv,
|
||
)
|
||
return summary_payload
|
||
|
||
|
||
def write_incremental_combined_outputs(
|
||
output_root: Path,
|
||
data_yaml: Path,
|
||
split_path: Path,
|
||
image_root: Path,
|
||
summary_by_roi: dict[str, Any],
|
||
num_entries: int,
|
||
args: argparse.Namespace,
|
||
elapsed_minutes: float,
|
||
portrait_payload: Optional[dict[str, Any]] = None,
|
||
) -> None:
|
||
portrait_pending = bool(not args.skip_data_portrait and portrait_payload is None)
|
||
combined_payload = {
|
||
"data": str(data_yaml),
|
||
"inference_config": str(args.inference_config),
|
||
"split": args.split,
|
||
"split_path": str(split_path),
|
||
"image_root": str(image_root),
|
||
"num_entries": int(num_entries),
|
||
"elapsed_minutes": float(elapsed_minutes),
|
||
"edge_yaw_max_lateral_dist_m": float(args.edge_yaw_max_lateral_dist),
|
||
"yaw_compare_max_lateral_dist_m": float(args.yaw_compare_max_lateral_dist),
|
||
"yaw_compare_max_longitudinal_dist_m": float(DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M),
|
||
"rois": summary_by_roi,
|
||
}
|
||
if portrait_payload is not None:
|
||
portrait_summary = portrait_payload.get("summary") or {}
|
||
combined_payload["data_portrait"] = {
|
||
"split": portrait_payload.get("split", args.data_portrait_split),
|
||
"summary_path": portrait_payload.get("summary_path"),
|
||
"num_entries": int(portrait_summary.get("num_entries", 0) or 0),
|
||
"vehicles": int(portrait_summary.get("vehicles", 0) or 0),
|
||
"mapped_objects": int(portrait_summary.get("mapped_objects", 0) or 0),
|
||
}
|
||
write_json(output_root / "summary.json", combined_payload)
|
||
|
||
lines = [
|
||
"# Two-ROI Validation Error Analysis",
|
||
"",
|
||
f"- data: `{data_yaml}`",
|
||
f"- inference_config: `{args.inference_config}`",
|
||
f"- split: `{args.split}`",
|
||
f"- split_path: `{split_path}`",
|
||
f"- image_root: `{image_root}`",
|
||
f"- num_entries: {int(num_entries)}",
|
||
f"- elapsed_minutes: {elapsed_minutes:.2f}",
|
||
"",
|
||
]
|
||
if portrait_payload is not None:
|
||
portrait_summary = portrait_payload.get("summary") or {}
|
||
lines.extend(
|
||
[
|
||
"## Data Portrait",
|
||
"",
|
||
f"- status: ready",
|
||
f"- split: `{portrait_payload.get('split', args.data_portrait_split)}`",
|
||
f"- entries: {int(portrait_summary.get('num_entries', 0) or 0)}",
|
||
f"- vehicles: {int(portrait_summary.get('vehicles', 0) or 0)}",
|
||
f"- mapped_objects: {int(portrait_summary.get('mapped_objects', 0) or 0)}",
|
||
f"- summary_json: `{portrait_payload.get('summary_path', 'n/a')}`",
|
||
"",
|
||
]
|
||
)
|
||
elif portrait_pending:
|
||
lines.extend(
|
||
[
|
||
"## Data Portrait",
|
||
"",
|
||
f"- status: building",
|
||
f"- split: `{args.data_portrait_split}`",
|
||
"",
|
||
]
|
||
)
|
||
for roi_name, payload in summary_by_roi.items():
|
||
overall = payload["overall"]
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
lines.extend(
|
||
[
|
||
f"## {roi_name.upper()}",
|
||
"",
|
||
f"- matched_2d: {overall['matched_2d']}",
|
||
f"- matched_3d: {overall['matched_3d']}",
|
||
f"- recommended_2d_confidence: {format_float(to_float(threshold_advice_2d.get('recommended_confidence')), 3)}",
|
||
f"- precision_recall_f1_2d: {format_percent(overall.get('precision_2d'))} / {format_percent(overall.get('recall_2d'))} / {format_percent(overall.get('f1_2d'))}",
|
||
f"- yaw_mae_deg: {overall['yaw_mae_deg']}",
|
||
f"- x_abs_mae_m: {overall['x_abs_mae_m']}",
|
||
f"- z_abs_mae_m: {overall['z_abs_mae_m']}",
|
||
f"- yaw_compare_count(gt_x in [-{args.yaw_compare_max_lateral_dist},{args.yaw_compare_max_lateral_dist})m, 5m bins): {overall['yaw_compare_count']}",
|
||
f"- direct_regression_yaw_mae_deg: {overall['direct_regression_yaw_mae_deg']}",
|
||
f"- edge_based_yaw_mae_deg: {overall['edge_based_yaw_mae_deg']}",
|
||
f"- summary: `{output_root / roi_name / 'summary.md'}`",
|
||
"",
|
||
]
|
||
)
|
||
(output_root / "summary.md").write_text("\n".join(lines), encoding="utf-8")
|
||
write_combined_summary_zh(output_root / "summary_zh.md", data_yaml=data_yaml, split_path=split_path, image_root=image_root, summary_by_roi=summary_by_roi)
|
||
write_combined_html_report(
|
||
output_root / "report.html",
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
combined_payload=combined_payload,
|
||
summary_by_roi=summary_by_roi,
|
||
portrait_payload=portrait_payload,
|
||
portrait_pending=portrait_pending,
|
||
)
|
||
|
||
|
||
def write_run_manifest(
|
||
output_root: Path,
|
||
data_yaml: Path,
|
||
split_path: Path,
|
||
image_root: Path,
|
||
num_entries: int,
|
||
requested_rois: set[str],
|
||
model_paths: dict[str, str],
|
||
args: argparse.Namespace,
|
||
portrait_payload: Optional[dict[str, Any]] = None,
|
||
portrait_split_path: Optional[Path] = None,
|
||
portrait_entries: Optional[list[tuple[str, str]]] = None,
|
||
) -> None:
|
||
portrait_summary = portrait_payload.get("summary") or {} if portrait_payload is not None else {}
|
||
portrait_num_entries = int(portrait_summary.get("num_entries", len(portrait_entries or [])) or 0)
|
||
write_json(
|
||
output_root / "run_manifest.json",
|
||
{
|
||
"data": str(data_yaml),
|
||
"inference_config": str(args.inference_config),
|
||
"split": args.split,
|
||
"split_path": str(split_path),
|
||
"image_root": str(image_root),
|
||
"num_entries": int(num_entries),
|
||
"analyze_rois": sorted(requested_rois),
|
||
"models": model_paths,
|
||
"edge_yaw_max_lateral_dist_m": float(args.edge_yaw_max_lateral_dist),
|
||
"yaw_compare_max_lateral_dist_m": float(args.yaw_compare_max_lateral_dist),
|
||
"yaw_compare_max_longitudinal_dist_m": float(DEFAULT_YAW_COMPARE_MAX_LONGITUDINAL_DIST_M),
|
||
"data_portrait": (
|
||
None
|
||
if portrait_payload is None
|
||
else {
|
||
"split": str(portrait_payload.get("split", args.data_portrait_split)),
|
||
"split_path": str(portrait_split_path) if portrait_split_path is not None else None,
|
||
"num_entries": portrait_num_entries,
|
||
"summary_path": portrait_payload.get("summary_path"),
|
||
}
|
||
),
|
||
"args": vars(args),
|
||
},
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
args = parse_args()
|
||
populate_two_roi_inference_args(args)
|
||
if args.torch_threads and args.torch_threads > 0:
|
||
torch.set_num_threads(int(args.torch_threads))
|
||
torch.set_num_interop_threads(max(1, min(int(args.torch_threads), 4)))
|
||
|
||
data_yaml = Path(args.data).resolve()
|
||
data_cfg = load_yaml(data_yaml)
|
||
dataset_root = data_cfg.get("path")
|
||
split_path = resolve_data_path(data_yaml, dataset_root, data_cfg.get(args.split))
|
||
entries = load_split_entries(
|
||
split_path,
|
||
max_samples=args.max_samples,
|
||
sample_selection=str(args.sample_selection),
|
||
sample_random_seed=int(args.sample_random_seed),
|
||
)
|
||
image_root = resolve_data_path(data_yaml, None, dataset_root)
|
||
|
||
args.roi0_data = args.roi0_data or args.data
|
||
args.roi1_data = args.roi1_data or args.data
|
||
requested_rois = {roi.lower() for roi in args.analyze_rois}
|
||
context = build_two_roi_inference_context_from_args(args, requested_rois=requested_rois)
|
||
|
||
output_root = Path(args.output_root).resolve()
|
||
output_root.mkdir(parents=True, exist_ok=True)
|
||
|
||
class_map = {str(key): int(value) for key, value in (data_cfg.get("class_map") or {}).items()}
|
||
difficulty_weights = [float(v) for v in data_cfg.get("difficulty_weights", [1.0, 1.0, 1.0, 1.0])]
|
||
face_3d_classes = set(int(v) for v in data_cfg.get("face_3d_classes", []))
|
||
complete_3d_classes = set(int(v) for v in data_cfg.get("complete_3d_classes", []))
|
||
include_classes = None if args.classes is None else set(int(v) for v in args.classes)
|
||
min_wh_px = float(data_cfg.get("min_wh", 2.0))
|
||
ori_img_size_cfg = data_cfg.get("ori_img_size", [1920, 1080])
|
||
ori_img_size = (
|
||
int(ori_img_size_cfg[0]) if isinstance(ori_img_size_cfg, (list, tuple)) and len(ori_img_size_cfg) > 0 else 1920,
|
||
int(ori_img_size_cfg[1]) if isinstance(ori_img_size_cfg, (list, tuple)) and len(ori_img_size_cfg) > 1 else 1080,
|
||
)
|
||
class_names = infer_class_name_map(class_map, names_to_dict(context.roi_models[0].names) if context.roi_models else None)
|
||
model_paths = {bundle.spec.name.lower(): bundle.spec.model_path for bundle in context.roi_models}
|
||
|
||
start = time.time()
|
||
summary_by_roi = {}
|
||
for bundle in context.roi_models:
|
||
print(f"\nAnalyzing {bundle.spec.name} on {len(entries)} {args.split} samples...")
|
||
summary_by_roi[bundle.spec.name.lower()] = run_roi_analysis(
|
||
bundle=bundle,
|
||
entries=entries,
|
||
image_root=image_root,
|
||
class_map=class_map,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
classes=include_classes,
|
||
min_wh_px=min_wh_px,
|
||
face_visibility_score_thresh=float(args.face_visibility_score_thresh),
|
||
yaw_bad_threshold_deg=float(args.yaw_bad_threshold_deg),
|
||
edge_yaw_max_lateral_dist_m=float(args.edge_yaw_max_lateral_dist),
|
||
yaw_compare_max_lateral_dist_m=float(args.yaw_compare_max_lateral_dist),
|
||
horizontal_bad_threshold_m=float(args.horizontal_bad_threshold_m),
|
||
vertical_bad_threshold_m=float(args.vertical_bad_threshold_m),
|
||
topk_badcases=int(args.topk_badcases),
|
||
per_class_badcases=int(args.per_class_badcases),
|
||
error_bin_badcases=int(args.error_bin_badcases),
|
||
error_bin_samples_per_bin=int(args.error_bin_samples_per_bin),
|
||
badcase_random_seed=int(args.badcase_random_seed),
|
||
output_root=output_root,
|
||
log_every=int(args.log_every),
|
||
split_file=str(split_path),
|
||
inference_batch_size=int(args.inference_batch_size),
|
||
)
|
||
write_incremental_combined_outputs(
|
||
output_root=output_root,
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
summary_by_roi=summary_by_roi,
|
||
num_entries=len(entries),
|
||
args=args,
|
||
elapsed_minutes=(time.time() - start) / 60.0,
|
||
portrait_payload=None,
|
||
)
|
||
print(f"Partial combined report updated: {output_root / 'report.html'}")
|
||
|
||
kpi_elapsed_minutes = (time.time() - start) / 60.0
|
||
write_incremental_combined_outputs(
|
||
output_root=output_root,
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
summary_by_roi=summary_by_roi,
|
||
num_entries=len(entries),
|
||
args=args,
|
||
elapsed_minutes=kpi_elapsed_minutes,
|
||
portrait_payload=None,
|
||
)
|
||
write_run_manifest(
|
||
output_root=output_root,
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
num_entries=len(entries),
|
||
requested_rois=requested_rois,
|
||
model_paths=model_paths,
|
||
args=args,
|
||
)
|
||
print(f"\nKPI report ready: {output_root / 'report.html'}")
|
||
|
||
portrait_payload = None
|
||
portrait_split_path = None
|
||
portrait_entries = None
|
||
if not args.skip_data_portrait:
|
||
portrait_payload = load_existing_data_portrait(output_root, str(args.data_portrait_split))
|
||
if portrait_payload is not None:
|
||
portrait_split_path_raw = portrait_payload.get("split_path")
|
||
portrait_split_path = None if not portrait_split_path_raw else Path(str(portrait_split_path_raw))
|
||
print(
|
||
f"\nReusing existing {args.data_portrait_split} data portrait from "
|
||
f"{portrait_payload.get('summary_path', output_root / f'{str(args.data_portrait_split).lower()}_portrait' / 'summary.json')}."
|
||
)
|
||
else:
|
||
portrait_split_value = data_cfg.get(args.data_portrait_split)
|
||
if portrait_split_value is None:
|
||
print(f"\nSkipping data portrait because split `{args.data_portrait_split}` is not defined in {data_yaml}.")
|
||
else:
|
||
portrait_split_path = resolve_data_path(data_yaml, dataset_root, portrait_split_value)
|
||
portrait_entries = load_split_entries(
|
||
portrait_split_path,
|
||
max_samples=int(args.data_portrait_max_samples),
|
||
sample_selection=str(args.data_portrait_sample_selection),
|
||
sample_random_seed=int(args.data_portrait_random_seed),
|
||
)
|
||
print(f"\nBuilding {args.data_portrait_split} data portrait on {len(portrait_entries)} samples...")
|
||
portrait_payload = build_data_portrait(
|
||
entries=portrait_entries,
|
||
split_name=str(args.data_portrait_split),
|
||
split_path=portrait_split_path,
|
||
image_root=image_root,
|
||
output_root=output_root,
|
||
class_map=class_map,
|
||
class_names=class_names,
|
||
difficulty_weights=difficulty_weights,
|
||
face_3d_classes=face_3d_classes,
|
||
complete_3d_classes=complete_3d_classes,
|
||
ori_img_size=ori_img_size,
|
||
face_visibility_score_thresh=float(args.face_visibility_score_thresh),
|
||
log_every=int(args.log_every),
|
||
workers=int(args.data_portrait_workers),
|
||
)
|
||
|
||
write_run_manifest(
|
||
output_root=output_root,
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
num_entries=len(entries),
|
||
requested_rois=requested_rois,
|
||
model_paths=model_paths,
|
||
args=args,
|
||
portrait_payload=portrait_payload,
|
||
portrait_split_path=portrait_split_path,
|
||
portrait_entries=portrait_entries,
|
||
)
|
||
|
||
total_minutes = (time.time() - start) / 60.0
|
||
write_incremental_combined_outputs(
|
||
output_root=output_root,
|
||
data_yaml=data_yaml,
|
||
split_path=split_path,
|
||
image_root=image_root,
|
||
summary_by_roi=summary_by_roi,
|
||
num_entries=len(entries),
|
||
args=args,
|
||
elapsed_minutes=total_minutes,
|
||
portrait_payload=portrait_payload,
|
||
)
|
||
|
||
print("\nAnalysis complete.")
|
||
print(f"Output root: {output_root}")
|
||
print(f"HTML report: {output_root / 'report.html'}")
|
||
if portrait_payload is not None:
|
||
print(f"Data portrait summary: {portrait_payload.get('summary_path')}")
|
||
for roi_name, payload in summary_by_roi.items():
|
||
overall = payload["overall"]
|
||
threshold_advice_2d = payload.get("threshold_advice_2d") or {}
|
||
print(
|
||
f"[{roi_name}] matched_3d={overall['matched_3d']} yaw_mae_deg={overall['yaw_mae_deg']} "
|
||
f"yaw_compare_count={overall['yaw_compare_count']} edge_based_mae_deg={overall['edge_based_yaw_mae_deg']} "
|
||
f"x_bad={overall['x_bad_count']} z_bad={overall['z_bad_count']} "
|
||
f"recommended_2d_conf={format_float(to_float(threshold_advice_2d.get('recommended_confidence')), 3)}"
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|