302 lines
13 KiB
Python
Executable File
302 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
YOLOv5-3D Model Evaluation Script
|
|
|
|
This script evaluates 2D and 3D detection performance of YOLOv5-3D model.
|
|
It computes metrics including Precision, Recall, AP, mAP for 2D detection,
|
|
and lateral/longitudinal errors and heading errors for 3D detection.
|
|
|
|
Usage:
|
|
# Basic usage with paths
|
|
python eval_tools/eval.py --det-path /path/to/detections --gt-path /path/to/labels --output-dir results
|
|
|
|
# With configuration file
|
|
python eval_tools/eval.py --config eval_tools/configs/eval_config.yaml
|
|
|
|
# Evaluate 2D only
|
|
python eval_tools/eval.py --det-path /path/to/detections --gt-path /path/to/labels --eval-2d-only
|
|
|
|
# Evaluate 3D only
|
|
python eval_tools/eval.py --det-path /path/to/detections --gt-path /path/to/labels --eval-3d-only
|
|
"""
|
|
|
|
import argparse
|
|
import sys
|
|
import yaml
|
|
from pathlib import Path
|
|
from datetime import datetime
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from eval_tools.evaluator import Evaluator
|
|
|
|
|
|
def load_config(config_path):
|
|
"""Load configuration from YAML file."""
|
|
with open(config_path, 'r') as f:
|
|
config = yaml.safe_load(f)
|
|
|
|
# Replace timestamp placeholders in paths
|
|
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
if 'output' in config and 'save_path' in config['output']:
|
|
config['output']['save_path'] = config['output']['save_path'].replace('{timestamp}', timestamp)
|
|
if 'dataset' in config:
|
|
if 'det_path' in config['dataset']:
|
|
config['dataset']['det_path'] = config['dataset']['det_path'].replace('{timestamp}', timestamp)
|
|
if 'gt_path' in config['dataset']:
|
|
config['dataset']['gt_path'] = config['dataset']['gt_path'].replace('{timestamp}', timestamp)
|
|
|
|
return config
|
|
|
|
|
|
def parse_arguments():
|
|
"""Parse command line arguments."""
|
|
parser = argparse.ArgumentParser(
|
|
description='Evaluate YOLOv5-3D model detection results',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Basic evaluation
|
|
python eval_tools/eval.py --det-path /data/detections --gt-path /data/labels --output-dir results
|
|
|
|
# With custom image size
|
|
python eval_tools/eval.py --det-path /data/detections --gt-path /data/labels --img-width 3840 --img-height 2160
|
|
|
|
# Using config file
|
|
python eval_tools/eval.py --config eval_tools/configs/eval_config.yaml
|
|
|
|
# 2D evaluation only
|
|
python eval_tools/eval.py --det-path /data/detections --gt-path /data/labels --eval-2d-only
|
|
"""
|
|
)
|
|
|
|
# Path arguments
|
|
parser.add_argument('--det-path', type=str,
|
|
help='Path to detection results root directory (contains case folders)')
|
|
parser.add_argument('--gt-path', type=str,
|
|
help='Path to ground truth labels root directory (contains case folders)')
|
|
parser.add_argument('--path-depth', type=int, default=1, choices=[1, 2],
|
|
help='Directory depth: 1 = det_path/case/txt_results, 2 = det_path/level1/case/txt_results (default: 1)')
|
|
parser.add_argument('--det-format', type=str, default='auto', choices=['auto', 'json', 'txt'],
|
|
help='Detection file format: auto (default, probe json_results/ then txt_results/), json, or txt')
|
|
parser.add_argument('--gt-format', type=str, default='auto', choices=['auto', 'json', 'txt'],
|
|
help='Ground truth file format: auto (default, probe labels_json/ then labels/), json, or txt')
|
|
parser.add_argument('--output-dir', type=str, default='eval_results',
|
|
help='Output directory for evaluation results (default: eval_results)')
|
|
|
|
# Config file
|
|
parser.add_argument('--config', type=str,
|
|
help='Path to YAML configuration file')
|
|
|
|
# Image parameters
|
|
parser.add_argument('--img-width', type=int, default=1920,
|
|
help='Image width in pixels (default: 1920)')
|
|
parser.add_argument('--img-height', type=int, default=1080,
|
|
help='Image height in pixels (default: 1080)')
|
|
|
|
# Evaluation options
|
|
parser.add_argument('--eval-2d-only', action='store_true',
|
|
help='Evaluate 2D detection only')
|
|
parser.add_argument('--eval-3d-only', action='store_true',
|
|
help='Evaluate 3D detection only')
|
|
parser.add_argument('--iou-threshold', type=float, default=0.5,
|
|
help='IoU threshold for 2D matching (default: 0.5)')
|
|
parser.add_argument('--conf-threshold', type=float, default=0.5,
|
|
help='Confidence threshold for precision/recall (default: 0.5)')
|
|
parser.add_argument('--ap-method', type=str, default='voc2010',
|
|
choices=['voc2010', 'coco'],
|
|
help='AP calculation method (default: voc2010)')
|
|
parser.add_argument('--heading-tolerance', type=str, default='strict',
|
|
choices=['strict', 'relaxed', 'both'],
|
|
help='Heading error calculation mode: strict (default), relaxed (180° symmetry), or both')
|
|
parser.add_argument('--coord-system', type=str, default='camera',
|
|
choices=['camera', 'ego'],
|
|
help='3D evaluation coordinate system: camera (default) or ego')
|
|
parser.add_argument('--num-workers', type=int, default=None,
|
|
help='Number of parallel workers for evaluation (default: auto-detect)')
|
|
parser.add_argument('--roi-bottom-offset', type=int, default=None,
|
|
help='Pixels to trim from bottom edge of ROI (shifts y2 upward, default: from config or 0)')
|
|
parser.add_argument('--save-detailed-matches', action='store_true',
|
|
help='Save detailed 3D match information for common-match comparison')
|
|
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
"""Main evaluation function."""
|
|
args = parse_arguments()
|
|
|
|
# Load configuration
|
|
config = {}
|
|
if args.config:
|
|
print(f"Loading configuration from: {args.config}")
|
|
config = load_config(args.config)
|
|
|
|
# Override with command line arguments if provided
|
|
if args.det_path:
|
|
config.setdefault('dataset', {})['det_path'] = args.det_path
|
|
if args.gt_path:
|
|
config.setdefault('dataset', {})['gt_path'] = args.gt_path
|
|
if args.path_depth != 1:
|
|
config.setdefault('dataset', {})['path_depth'] = args.path_depth
|
|
if args.det_format != 'auto':
|
|
config.setdefault('dataset', {})['det_format'] = args.det_format
|
|
if args.gt_format != 'auto':
|
|
config.setdefault('dataset', {})['gt_format'] = args.gt_format
|
|
# Only override output_dir if explicitly provided (not default value)
|
|
if args.output_dir != 'eval_results':
|
|
config.setdefault('output', {})['save_path'] = args.output_dir
|
|
|
|
# Override image size if explicitly provided
|
|
if args.img_width != 1920:
|
|
config.setdefault('image', {})['width'] = args.img_width
|
|
if args.img_height != 1080:
|
|
config.setdefault('image', {})['height'] = args.img_height
|
|
|
|
# Override matching threshold if explicitly provided
|
|
if args.iou_threshold != 0.5:
|
|
config.setdefault('matching', {})['iou_threshold'] = args.iou_threshold
|
|
|
|
# Override 2D metrics parameters if explicitly provided
|
|
if args.conf_threshold != 0.5:
|
|
config.setdefault('metrics_2d', {})['conf_threshold'] = args.conf_threshold
|
|
if args.ap_method != 'voc2010':
|
|
config.setdefault('metrics_2d', {})['ap_method'] = args.ap_method
|
|
|
|
# Override 3D metrics parameters if explicitly provided
|
|
if args.heading_tolerance != 'strict':
|
|
config.setdefault('metrics_3d', {})['heading_tolerance'] = args.heading_tolerance
|
|
if args.coord_system != 'camera':
|
|
config.setdefault('metrics_3d', {})['coordinate_system'] = args.coord_system
|
|
|
|
# Override roi_bottom_offset if explicitly provided
|
|
if args.roi_bottom_offset is not None:
|
|
config.setdefault('roi_gt', {})['roi_bottom_offset'] = args.roi_bottom_offset
|
|
else:
|
|
# Build config from command line arguments
|
|
if not args.det_path or not args.gt_path:
|
|
print("Error: --det-path and --gt-path are required when not using --config")
|
|
sys.exit(1)
|
|
|
|
config = {
|
|
'dataset': {
|
|
'det_path': args.det_path,
|
|
'gt_path': args.gt_path,
|
|
'path_depth': args.path_depth,
|
|
'det_format': args.det_format,
|
|
'gt_format': args.gt_format,
|
|
},
|
|
'image': {
|
|
'width': args.img_width,
|
|
'height': args.img_height
|
|
},
|
|
'matching': {
|
|
'iou_threshold': args.iou_threshold
|
|
},
|
|
'metrics_2d': {
|
|
'enabled': not args.eval_3d_only,
|
|
'conf_threshold': args.conf_threshold,
|
|
'ap_method': args.ap_method
|
|
},
|
|
'metrics_3d': {
|
|
'enabled': not args.eval_2d_only,
|
|
'heading_tolerance': args.heading_tolerance,
|
|
'coordinate_system': args.coord_system,
|
|
},
|
|
'output': {
|
|
'save_path': args.output_dir
|
|
}
|
|
}
|
|
|
|
if args.roi_bottom_offset is not None:
|
|
config.setdefault('roi_gt', {})['roi_bottom_offset'] = args.roi_bottom_offset
|
|
|
|
# Set evaluation flags
|
|
config['eval_2d'] = config.get('metrics_2d', {}).get('enabled', True)
|
|
config['eval_3d'] = config.get('metrics_3d', {}).get('enabled', True)
|
|
|
|
if args.eval_2d_only:
|
|
config['eval_2d'] = True
|
|
config['eval_3d'] = False
|
|
elif args.eval_3d_only:
|
|
config['eval_2d'] = False
|
|
config['eval_3d'] = True
|
|
|
|
# Extract paths
|
|
det_path = config['dataset']['det_path']
|
|
gt_path = config['dataset']['gt_path']
|
|
path_depth = config['dataset'].get('path_depth', 1) # Default to 1-level structure
|
|
det_format = config['dataset'].get('det_format', 'auto')
|
|
gt_format = config['dataset'].get('gt_format', 'auto')
|
|
det_subdir = config['dataset'].get('det_subdir')
|
|
output_dir = config['output']['save_path']
|
|
|
|
img_width = config.get('image', {}).get('width', 1920)
|
|
img_height = config.get('image', {}).get('height', 1080)
|
|
iou_threshold = config.get('matching', {}).get('iou_threshold', 0.5)
|
|
|
|
# Get num_workers from config if not specified in command line
|
|
num_workers = args.num_workers
|
|
if num_workers is None and 'performance' in config:
|
|
num_workers = config['performance'].get('num_workers', None)
|
|
|
|
print("="*80)
|
|
print("YOLOv5-3D Model Evaluation")
|
|
print("="*80)
|
|
print(f"Detection path: {det_path}")
|
|
print(f"Ground truth path: {gt_path}")
|
|
print(f"Path depth: {path_depth}")
|
|
print(f"Detection format: {det_format}")
|
|
print(f"Ground truth format: {gt_format}")
|
|
if det_subdir:
|
|
print(f"Detection subdirectory: {det_subdir}")
|
|
print(f"Output directory: {output_dir}")
|
|
print(f"Image size: {img_width}x{img_height}")
|
|
print(f"IoU threshold: {iou_threshold}")
|
|
print(f"Confidence threshold: {config.get('metrics_2d', {}).get('conf_threshold', 0.5)}")
|
|
print(f"AP method: {config.get('metrics_2d', {}).get('ap_method', 'voc2010')}")
|
|
heading_tolerance = config.get('metrics_3d', {}).get('heading_tolerance', 'strict')
|
|
coord_system = config.get('metrics_3d', {}).get('coordinate_system', 'camera')
|
|
print(f"Heading tolerance: {heading_tolerance}")
|
|
print(f"3D coordinate system: {coord_system}")
|
|
roi_bottom_offset_val = config.get('roi_gt', {}).get('roi_bottom_offset', 0)
|
|
if config.get('roi_gt', {}).get('enabled', False):
|
|
print(f"ROI bottom offset: {roi_bottom_offset_val}")
|
|
if num_workers:
|
|
print(f"Number of workers: {num_workers}")
|
|
print(f"Evaluate 2D: {config['eval_2d']}")
|
|
print(f"Evaluate 3D: {config['eval_3d']}")
|
|
if args.save_detailed_matches:
|
|
print(f"Save detailed matches: Yes")
|
|
print("="*80)
|
|
|
|
# Initialize evaluator
|
|
evaluator = Evaluator(
|
|
config=config,
|
|
iou_threshold=iou_threshold,
|
|
num_workers=num_workers,
|
|
save_detailed_matches=args.save_detailed_matches
|
|
)
|
|
|
|
# Load data
|
|
print("\nLoading data...")
|
|
evaluator.load_data_from_paths(det_path, gt_path, img_width, img_height, path_depth,
|
|
det_format=det_format, gt_format=gt_format)
|
|
|
|
if len(evaluator.image_pairs) == 0:
|
|
print("Error: No image pairs found for evaluation!")
|
|
sys.exit(1)
|
|
|
|
# Run evaluation
|
|
results = evaluator.evaluate()
|
|
|
|
# Generate and save report
|
|
evaluator.generate_report(results, output_dir)
|
|
|
|
print("\n✓ Evaluation completed successfully!")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|