#!/usr/bin/env python3
"""
Ground Joint 2D+3D Detection Training Script for YOLO26
Supports single-GPU and multi-GPU distributed training with 3D loss ramping.

Usage:
    # Single GPU with pretrained 2D weights
    python train_mono3d.py --model yolo26s --pretrained yolo26s-pretrain.pt --epochs 100 --device 0

    # Multi-GPU (DDP) with pretrained 2D weights
    python -m torch.distributed.run --nproc_per_node 4 train_mono3d.py --model yolo26s --pretrained yolo26s-pretrain.pt --epochs 100

    # Train from scratch (no pretrained weights)
    python train_mono3d.py --epochs 100 --device 0

    # Resume from checkpoint
    python train_mono3d.py --resume runs/detect/train_mono3d/weights/last.pt
"""

import argparse
import os
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Optional, Tuple
from uuid import uuid4

from torch.distributed.elastic.multiprocessing.errors import record

os.environ['TORCH_CACHE_DIR'] = './torch_cache'
os.makedirs(os.environ['TORCH_CACHE_DIR'], exist_ok=True)

# Add ultralytics to path if needed
FILE = Path(__file__).resolve()
ROOT = FILE.parents[0]
if str(ROOT) not in sys.path:
    sys.path.append(str(ROOT))

from ultralytics import settings
from ultralytics.models.yolo.detect.train import Ground3DDetectionTrainer
from ultralytics.utils import LOGGER, YAML, colorstr
from ultralytics.utils.dist import get_distributed_run_timestamp

# Enable TensorBoard logging
settings.update({"tensorboard": True})

GROUND3D_REQUIRED_CONFIG_KEYS = ("path", "ori_img_size", "roi", "virtual_fx", "virtual_camera_prob", "crop_center_mode")
GROUND3D_VALID_CROP_CENTER_MODES = {"cxvy", "vxvy"}
GROUND3D_MODEL_VARIANTS = {f"yolo26{scale}": scale for scale in "nslmx"}
GROUND3D_DEFAULT_MODEL_CFG = "ultralytics/cfg/models/26/yolo26-3d.yaml"


def _validate_ground3d_pair(cfg: dict[str, Any], key: str, context: str) -> None:
    """Validate that a Ground3D size-like config field is an integer pair."""
    value = cfg.get(key)
    if not isinstance(value, (list, tuple)) or len(value) != 2:
        raise ValueError(f"{context} must define '{key}' as a 2-item list/tuple, but got: {value!r}")
    if any(int(v) <= 0 for v in value):
        raise ValueError(f"{context} must define positive values for '{key}', but got: {value!r}")


def _validate_ground3d_config(cfg: dict[str, Any], context: str) -> None:
    """Validate required Ground3D config fields before training starts."""
    missing = [key for key in GROUND3D_REQUIRED_CONFIG_KEYS if key not in cfg or cfg[key] is None]
    if missing:
        missing_str = ", ".join(missing)
        raise ValueError(f"{context} is missing required Ground3D config field(s): {missing_str}.")

    if not str(cfg["path"]).strip():
        raise ValueError(f"{context} must define a non-empty 'path'.")

    _validate_ground3d_pair(cfg, "ori_img_size", context)
    _validate_ground3d_pair(cfg, "roi", context)

    if float(cfg["virtual_fx"]) <= 0:
        raise ValueError(f"{context} must define a positive 'virtual_fx', but got: {cfg['virtual_fx']!r}.")
    try:
        float(cfg["virtual_camera_prob"])
    except (TypeError, ValueError) as e:
        raise ValueError(
            f"{context} must define 'virtual_camera_prob' as a numeric value, but got: {cfg['virtual_camera_prob']!r}."
        ) from e

    crop_center_mode = cfg["crop_center_mode"]
    if crop_center_mode not in GROUND3D_VALID_CROP_CENTER_MODES:
        valid_modes = ", ".join(sorted(GROUND3D_VALID_CROP_CENTER_MODES))
        raise ValueError(
            f"{context} must define 'crop_center_mode' as one of {{{valid_modes}}}, but got: {crop_center_mode!r}."
        )


def build_run_name(base_name: str, resume: str, world_size: int, rank: int) -> str:
    """Build a timestamped run name that stays consistent across distributed ranks."""
    if resume:
        resume_path = Path(resume)
        if resume_path.suffix == ".pt" and len(resume_path.parents) >= 2:
            return resume_path.parents[1].name
        return base_name

    timestamp = get_distributed_run_timestamp(base_name, world_size, rank)
    return f"{base_name}_{timestamp}"


def resolve_ground3d_model(model_arg: str) -> Tuple[str, Optional[str]]:
    """Resolve shorthand variants like 'yolo26s' to the shared 3D YAML plus an explicit scale."""
    model_arg = model_arg.strip()
    if re.fullmatch(r"yolo26[nslmx]", model_arg):
        return GROUND3D_DEFAULT_MODEL_CFG, GROUND3D_MODEL_VARIANTS[model_arg]
    return str(Path(model_arg).expanduser()), None


def resolve_data_yaml_for_roi(data_path: str, roi_name: Optional[str]) -> Tuple[str, Optional[str]]:
    """Resolve a dataset YAML to a temporary file with the selected ROI preset applied."""
    data_path = str(Path(data_path).expanduser())
    if Path(data_path).suffix not in {".yaml", ".yml"}:
        if roi_name:
            raise ValueError("--roi requires --data to point to a dataset YAML file.")
        return data_path, None

    data_cfg = YAML.load(data_path)
    roi_configs = data_cfg.get("roi_configs")
    default_roi = data_cfg.get("default_roi")
    selected_roi = roi_name or default_roi

    if selected_roi is None:
        return data_path, None
    if not isinstance(roi_configs, dict) or not roi_configs:
        raise ValueError(f"Dataset YAML '{data_path}' does not define any 'roi_configs' presets.")
    if selected_roi not in roi_configs:
        available = ", ".join(sorted(roi_configs))
        raise ValueError(
            f"Unknown ROI preset '{selected_roi}' for dataset '{data_path}'. Available presets: {available}."
        )

    roi_overrides = roi_configs[selected_roi]
    if not isinstance(roi_overrides, dict):
        raise ValueError(f"ROI preset '{selected_roi}' in '{data_path}' must be a mapping.")

    resolved_cfg: dict[str, Any] = dict(data_cfg)
    resolved_cfg.pop("default_roi", None)
    resolved_cfg.pop("roi_configs", None)
    resolved_cfg.update(roi_overrides)
    _validate_ground3d_config(resolved_cfg, f"Resolved dataset YAML '{data_path}' with ROI preset '{selected_roi}'")

    source_path = Path(data_path)
    resolved_path = (
        Path(tempfile.gettempdir())
        / "ultralytics_roi_configs"
        / f"{source_path.stem}.{selected_roi}.{os.getpid()}.{uuid4().hex}{source_path.suffix}"
    )
    header = (
        "# Auto-generated by train_mono3d.py\n"
        f"# Source dataset YAML: {source_path.resolve()}\n"
        f"# Selected ROI preset: {selected_roi}\n\n"
    )
    YAML.save(file=resolved_path, data=resolved_cfg, header=header)
    return str(resolved_path), selected_roi


def parse_args():
    """Parse command line arguments."""
    parser = argparse.ArgumentParser(description="Train YOLO26 on Ground Joint 2D+3D Detection Dataset")                          # description 是帮助信息标题。

    # Model and data
    parser.add_argument(
        "--model", type=str, default=GROUND3D_DEFAULT_MODEL_CFG,
        help="Model config YAML path or shorthand variant (yolo26n/yolo26s/yolo26m/yolo26l/yolo26x).",
    )                                              # --model ultralytics/cfg/models/26/yolo26-3d.yaml  或者  --model yolo26s
    parser.add_argument("--data", type=str, default="ultralytics/cfg/datasets/mono3d_ground.yaml", help="Dataset YAML path")
    parser.add_argument(
        "--roi",
        type=str,
        default=None,
        help="Named ROI preset from the dataset YAML 'roi_configs' section (for example: roi0 or roi1)",
    )

    # Training hyperparameters
    parser.add_argument("--epochs", type=int, default=100, help="Number of training epochs")
    parser.add_argument("--batch", type=int, default=16, help="Batch size per GPU")
    parser.add_argument("--imgsz", type=str, default="704,352", help="Image size (int or w,h e.g. 704,352)")
    parser.add_argument("--device", type=str, default="0,1,2,3", help="Device to use (e.g., 0 or 0,1,2,3 or cpu)")

    # Optimizer
    parser.add_argument("--optimizer", type=str, default="auto", choices=["SGD", "Adam", "AdamW", "auto"], help="Optimizer")
    parser.add_argument("--lr0", type=float, default=0.01, help="Initial learning rate")
    parser.add_argument("--lrf", type=float, default=0.01, help="Final learning rate factor")
    parser.add_argument("--momentum", type=float, default=0.937, help="SGD momentum/Adam beta1")
    parser.add_argument("--weight_decay", type=float, default=0.0005, help="Optimizer weight decay")
    parser.add_argument("--warmup_epochs", type=float, default=3.0, help="Warmup epochs (2D-only stage if loss_3d_warmup_epochs is not set)")
    parser.add_argument("--warmup_momentum", type=float, default=0.8, help="Initial momentum during warmup")
    parser.add_argument("--warmup_bias_lr", type=float, default=0.1, help="Initial bias learning rate during warmup")
    parser.add_argument("--e2e_o2m_start", type=float, default=0.8, help="Initial one-to-many loss weight for end-to-end training")
    parser.add_argument("--e2e_o2m_final", type=float, default=0.1, help="Final one-to-many loss weight after decay")
    parser.add_argument(
        "--e2e_o2m_decay_epochs",
        type=float,
        default=None,
        help="Epochs used to decay one-to-many weight to the final value; defaults to epochs - 1",
    )
    parser.add_argument("--loss_3d_warmup_epochs", type=float, default=None, help="Epochs before enabling 3D loss; defaults to warmup_epochs")
    parser.add_argument("--loss_3d_ramp_epochs", type=float, default=10.0, help="Epochs used to ramp 3D loss weight to its max value")
    parser.add_argument("--loss_3d_weight_max", type=float, default=0.1, help="Maximum 3D loss weight multiplier")
    parser.add_argument(
        "--face_visibility_score_thresh",
        type=float,
        default=None,
        help="Override visible-face score threshold used by 3D face supervision and visible-face metrics",
    )

    # Loss weights
    parser.add_argument("--box", type=float, default=7.5, help="Box loss gain")
    parser.add_argument("--cls", type=float, default=0.5, help="Class loss gain")
    parser.add_argument("--dfl", type=float, default=1.5, help="DFL loss gain")                   # DFL loss 权重。        注意当前 yolo26-3d.yaml 里 reg_max=1，DFL 实际弱化/关闭，但参数仍保留兼容。

    # Augmentation
    parser.add_argument("--hsv_h", type=float, default=0.015, help="HSV-Hue augmentation")
    parser.add_argument("--hsv_s", type=float, default=0.7, help="HSV-Saturation augmentation")
    parser.add_argument("--hsv_v", type=float, default=0.4, help="HSV-Value augmentation")
    parser.add_argument("--degrees", type=float, default=0.0, help="Rotation augmentation (degrees)")
    parser.add_argument("--translate", type=float, default=0.1, help="Translation augmentation")
    parser.add_argument("--scale", type=float, default=0.5, help="Scale augmentation")
    parser.add_argument("--shear", type=float, default=0.0, help="Shear augmentation (degrees)")
    parser.add_argument("--perspective", type=float, default=0.0, help="Perspective augmentation")
    parser.add_argument("--flipud", type=float, default=0.0, help="Vertical flip probability")
    parser.add_argument("--fliplr", type=float, default=0.5, help="Horizontal flip probability")
    parser.add_argument("--mosaic", type=float, default=1.0, help="Mosaic augmentation probability")
    parser.add_argument("--mixup", type=float, default=0.0, help="Mixup augmentation probability")
    parser.add_argument("--copy_paste", type=float, default=0.0, help="Copy-paste augmentation probability")

    # Training settings
    parser.add_argument("--patience", type=int, default=100, help="Early stopping patience")                                                     # early stopping 的 patience。如果验证指标长期不提升，最多等 100 个 epoch。
    parser.add_argument("--save_period", type=int, default=1, help="Save checkpoint every x epochs (default: every epoch, -1 to disable)")
    parser.add_argument("--cache", type=str, default="", choices=["", "ram", "disk"], help="Cache images")                               # 是否缓存图片
    parser.add_argument("--workers", type=int, default=16, help="Number of dataloader workers")
    parser.add_argument("--project", type=str, default=None, help="Project directory")                                                   # 训练结果保存的 project 目录
    parser.add_argument("--name", type=str, default="train_mono3d", help="Experiment name")
    parser.add_argument("--exp_dir", type=str, default="",
        help="Exact experiment directory to use. Bypasses distributed run-name timestamp sync.",
    )                      # 如果传这个，就直接用该目录保存结果，不再走默认 project/name 逻辑
    parser.add_argument("--exist_ok", action="store_true", help="Overwrite existing experiment")
    parser.add_argument("--pretrained", type=str, default="", help="Pretrained 2D weights path (e.g. yolo26s-pretrain.pt). Scale extracted from filename.")
    parser.add_argument("--resume", type=str, default="", help="Resume training from checkpoint")
    parser.add_argument(
        "--strict-backbone-pretrained",
        dest="strict_backbone_pretrained",
        action=argparse.BooleanOptionalAction,
        default=True,
        help="Fail startup if any backbone parameter tensor can not be transferred from --pretrained.",
    )
    parser.add_argument("--amp", action="store_true", help="Enable automatic mixed precision")
    parser.add_argument("--fraction", type=float, default=1.0, help="Dataset fraction to use")                          # 只用 10% 数据，适合快速调试
    parser.add_argument("--profile", action="store_true", help="Profile ONNX and TensorRT speeds")
    parser.add_argument("--freeze", type=int, default=None, help="Freeze first n layers")
    parser.add_argument("--multi_scale", type=float, default=0.0, help="Multi-scale training range")

    # Validation
    parser.add_argument("--val", action="store_true", default=True, help="Validate during training")                    # 默认已经是 True，传 --val 也还是 True。也就是说这段代码没有提供一个直接关闭 val 的 --no-val
    parser.add_argument("--plots", action="store_true", default=True, help="Save plots during training")                # 是否保存训练/验证图
    parser.add_argument("--roi_metrics_only", action="store_true", help="Compute validation metrics using ROI samples only")    # 当 batch 里有 camera_mode 时，validation 可以跳过 virtual-camera 样本，只统计 ROI 样本的 metrics

    # Advanced
    parser.add_argument("--seed", type=int, default=0, help="Random seed")
    parser.add_argument("--deterministic", action="store_true", help="Use deterministic algorithms")
    parser.add_argument("--single_cls", action="store_true", help="Train as single-class dataset")                    # 是否按单类别训练，把所有类别合并成一个类
    parser.add_argument("--rect", action="store_true", help="Rectangular training")
    parser.add_argument("--cos_lr", action="store_true", help="Use cosine learning rate scheduler")
    parser.add_argument("--close_mosaic", type=int, default=10, help="Disable mosaic augmentation for final epochs")
    parser.add_argument("--nbs", type=int, default=64, help="Nominal batch size")
    parser.add_argument("--overlap_mask", action="store_true", default=True, help="Masks should overlap during training")
    parser.add_argument("--mask_ratio", type=int, default=4, help="Mask downsample ratio")
    parser.add_argument("--dropout", type=float, default=0.0, help="Dropout rate")

    return parser.parse_args()


@record                                                                                                          # 在 DDP / 分布式训练时，如果子进程报错，它能把异常栈记录得更完整，方便定位哪个 rank 出错
def main():
    """Main training function. 解析命令行参数 -> 处理模型/数据/ROI 配置 -> 组装 Ultralytics trainer 参数 -> 启动 Ground3DDetectionTrainer 训练"""
    args = parse_args()

    # Print banner
    LOGGER.info(colorstr("bright_blue", "bold", "\n" + "=" * 80))
    LOGGER.info(colorstr("bright_blue", "bold", "Ground Joint 2D+3D Detection Training with YOLO26"))
    LOGGER.info(colorstr("bright_blue", "bold", "=" * 80 + "\n"))

    # Confirm TensorBoard is enabled
    LOGGER.info(colorstr("bright_green", "TensorBoard logging enabled"))

    # Check for distributed training
    world_size = int(os.getenv("WORLD_SIZE", 1))
    rank = int(os.getenv("RANK", -1))
    local_rank = int(os.getenv("LOCAL_RANK", -1))

    if world_size > 1:
        LOGGER.info(f"Distributed training: world_size={world_size}, rank={rank}, local_rank={local_rank}")
    else:
        LOGGER.info("Single GPU training")

    # For 3D training: --model can be the 3D config yaml or a shorthand variant like "yolo26s".
    pretrained = args.pretrained if args.pretrained else False
    resolved_model_path, explicit_model_scale = resolve_ground3d_model(args.model)                                               # 如果是 yolo26s，resolve_ground3d_model() 会返回：  resolved_model_path = "ultralytics/cfg/models/26/yolo26-3d.yaml"  +  explicit_model_scale = "s"
    if pretrained and explicit_model_scale is None and not re.fullmatch(r"yolo26[nslmx].*", Path(str(pretrained)).stem):         # 如果用了 pretrained，但是 --model 没有显式写 yolo26n/s/l/m/x，并且 pretrained 文件名里也看不出 scale，就发 warning。
        LOGGER.warning(
            "Pretrained checkpoint name does not encode a YOLO26 scale. Pass '--model yolo26n/yolo26s/...' "
            "to lock the 3D architecture scale explicitly."
        )

    # Parse imgsz: "704,352" -> [704, 352], "640" -> 640
    if isinstance(args.imgsz, str) and "," in args.imgsz:
        imgsz = [int(x) for x in args.imgsz.split(",")]               # --imgsz 704,352  --> imgsz = [704, 352]
    else:
        imgsz = int(args.imgsz)

    if args.resume and args.exp_dir:                                  # 如果 --resume 断点恢复，训练目录已经由 checkpoint 决定；此时不能再手动指定 --exp_dir。
        raise ValueError("--exp_dir cannot be used with --resume because resume already determines the run directory.")

    exp_dir = str(Path(args.exp_dir).expanduser()) if args.exp_dir else ""
    run_name = Path(exp_dir).name if exp_dir else build_run_name(args.name, args.resume, world_size, rank)      # 如果指定了 exp_dir，实验名就是目录名。 否则调用 build_run_name()，通常会加时间戳，并保证 DDP 多个 rank 用同一个名字。      
    resolved_data_path, selected_roi = resolve_data_yaml_for_roi(args.data, args.roi)                           # --data ultralytics/cfg/datasets/mono3d_ground.yaml --roi roi1   ==>  --data ultralytics/cfg/datasets/mono3d_ground.yaml --roi roi1   selected_roi = "roi1"
 
    # Build overrides dictionary
    overrides = {
        "model": resolved_model_path,
        "data": resolved_data_path,
        "epochs": args.epochs,
        "batch": args.batch,
        "imgsz": imgsz,
        "device": args.device,
        "optimizer": args.optimizer,
        "lr0": args.lr0,
        "lrf": args.lrf,
        "momentum": args.momentum,
        "weight_decay": args.weight_decay,
        "warmup_epochs": args.warmup_epochs,
        "warmup_momentum": args.warmup_momentum,
        "warmup_bias_lr": args.warmup_bias_lr,
        "e2e_o2m_start": args.e2e_o2m_start,                         # YOLO26 end-to-end 训练里 one-to-many loss 的权重控制。0.8-->0.1
        "e2e_o2m_final": args.e2e_o2m_final,
        "loss_3d_ramp_epochs": args.loss_3d_ramp_epochs,             # 3D loss 不一定一开始就满权重参与，而是逐渐从 0 增长到最大值。10
        "loss_3d_weight_max": args.loss_3d_weight_max,               # 0.1
        "box": args.box,
        "cls": args.cls,
        "dfl": args.dfl,
        "hsv_h": args.hsv_h,
        "hsv_s": args.hsv_s,
        "hsv_v": args.hsv_v,
        "degrees": args.degrees,
        "translate": args.translate,
        "scale": args.scale,
        "shear": args.shear,
        "perspective": args.perspective,
        "flipud": args.flipud,
        "fliplr": args.fliplr,
        "mosaic": args.mosaic,
        "mixup": args.mixup,
        "copy_paste": args.copy_paste,
        "patience": args.patience,                             # early stopping 等待轮数。
        "save_period": args.save_period,                       # 每隔多少 epoch 保存 checkpoint。
        "cache": args.cache if args.cache else False,
        "workers": args.workers,
        "project": args.project,
        "name": run_name,
        "exist_ok": args.exist_ok,
        "pretrained": pretrained,
        "amp": args.amp,
        "fraction": args.fraction,
        "profile": args.profile,
        "multi_scale": args.multi_scale,
        "val": args.val,
        "plots": args.plots,
        "roi_metrics_only": args.roi_metrics_only,
        "seed": args.seed,
        "deterministic": args.deterministic,
        "single_cls": args.single_cls,
        "rect": args.rect,
        "cos_lr": args.cos_lr,
        "close_mosaic": args.close_mosaic,
        "nbs": args.nbs,
        "overlap_mask": args.overlap_mask,
        "mask_ratio": args.mask_ratio,
        "dropout": args.dropout,
    }

    # Add freeze if specified
    if args.freeze is not None:                   # 果用户传了 --freeze，加入冻结层数配置。
        overrides["freeze"] = args.freeze

    if args.loss_3d_warmup_epochs is not None:
        overrides["loss_3d_warmup_epochs"] = args.loss_3d_warmup_epochs
    if args.e2e_o2m_decay_epochs is not None:
        overrides["e2e_o2m_decay_epochs"] = args.e2e_o2m_decay_epochs
    if args.face_visibility_score_thresh is not None:
        overrides["face_visibility_score_thresh"] = args.face_visibility_score_thresh

    # Add resume if specified
    if args.resume:
        overrides["resume"] = args.resume
    if exp_dir:
        overrides["save_dir"] = exp_dir

    # Print configuration
    LOGGER.info(colorstr("bright_cyan", "\nTraining Configuration:"))
    LOGGER.info(f"  Model arg: {args.model}")
    LOGGER.info(f"  Model config: {resolved_model_path}")
    LOGGER.info(
        "  Model scale: "
        f"{explicit_model_scale if explicit_model_scale else 'infer from pretrained filename or YAML default'}"
    )
    LOGGER.info(f"  Pretrained weights: {pretrained if pretrained else 'None'}")
    LOGGER.info(f"  Strict pretrained backbone load: {args.strict_backbone_pretrained}")
    LOGGER.info(f"  Data: {args.data}")
    if selected_roi:
        LOGGER.info(f"  ROI preset: {selected_roi}")
        LOGGER.info(f"  Resolved data YAML: {resolved_data_path}")
    LOGGER.info(f"  Epochs: {args.epochs}")
    LOGGER.info(f"  Batch size: {args.batch}")
    LOGGER.info(f"  Image size: {args.imgsz}")
    LOGGER.info(f"  Device: {args.device if args.device else 'auto'}")
    LOGGER.info(f"  Workers: {args.workers}")
    LOGGER.info(f"  Cache: {args.cache if args.cache else 'False'}")
    LOGGER.info(f"  Run name: {run_name}")
    LOGGER.info(f"  Checkpoints: every {args.save_period} epoch(s)" if args.save_period > 0 else "  Checkpoints: last.pt and best.pt only")
    if exp_dir:
        LOGGER.info(f"  Experiment dir: {exp_dir}")
    LOGGER.info(f"  AMP: {args.amp}")
    LOGGER.info(f"  Optimizer: {args.optimizer}")
    LOGGER.info(f"  Learning rate: {args.lr0} -> {args.lr0 * args.lrf}")
    LOGGER.info(f"  Warmup: epochs={args.warmup_epochs}, momentum={args.warmup_momentum}, bias_lr={args.warmup_bias_lr}")
    LOGGER.info(
        "  E2E one-to-many weight: "
        f"start={args.e2e_o2m_start}, final={args.e2e_o2m_final}, "
        f"decay_epochs={args.e2e_o2m_decay_epochs if args.e2e_o2m_decay_epochs is not None else 'epochs-1'}"
    )
    LOGGER.info(f"  Loss weights: box={args.box}, cls={args.cls}, dfl={args.dfl}")
    loss_3d_warmup = args.loss_3d_warmup_epochs if args.loss_3d_warmup_epochs is not None else args.warmup_epochs
    LOGGER.info(
        f"  3D loss: warmup={loss_3d_warmup} epochs, ramp={args.loss_3d_ramp_epochs} epochs, max={args.loss_3d_weight_max}"
    )
    LOGGER.info(
        "  Face visibility score threshold: "
        f"{args.face_visibility_score_thresh if args.face_visibility_score_thresh is not None else 'default.yaml'}"
    )
    LOGGER.info("")

    # Initialize trainer
    LOGGER.info(colorstr("bright_green", "Initializing Ground3DDetectionTrainer..."))
    trainer = Ground3DDetectionTrainer(overrides=overrides)                # 进入 Ultralytics 训练框架，做很多初始化
    trainer.explicit_model_scale = explicit_model_scale                    # 把显式模型 scale 挂到 trainer 上。后面 Ground3DDetectionTrainer.get_model() 会用它来决定模型规模，比如 n/s/m/l/x。
    trainer.strict_backbone_pretrained = args.strict_backbone_pretrained   # 把严格预训练加载检查开关挂到 trainer 上。后面加载 pretrained 时会用。

    # Start training
    LOGGER.info(colorstr("bright_green", "Starting training...\n"))
    trainer.train()

    # Print completion message
    LOGGER.info(colorstr("bright_blue", "bold", "\n" + "=" * 80))
    LOGGER.info(colorstr("bright_blue", "bold", "Training completed!"))
    LOGGER.info(colorstr("bright_blue", "bold", "=" * 80 + "\n"))

    # Print results location
    if hasattr(trainer, "save_dir"):
        LOGGER.info(f"Results saved to: {trainer.save_dir}")
        LOGGER.info(f"Best weights: {trainer.save_dir / 'weights' / 'best.pt'}")
        LOGGER.info(f"Last weights: {trainer.save_dir / 'weights' / 'last.pt'}")


if __name__ == "__main__":
    main()