# 基于共同匹配集的3D指标对比方案 ## 问题分析 ### 当前问题 两个模型评测时匹配的样本数量不一致： - **Model A**: 匹配 440,408 个 vehicle - **Model B**: 匹配 491,768 个 vehicle - 差异：51,360 个样本（11.7%） **导致的问题**： - 无法确定性能差异是因为**模型质量**还是**匹配了不同的目标** - 比较的不是相同目标的预测质量 ### 解决方案 只比较两个模型**都成功匹配到GT的目标**，确保对比的是相同目标的3D性能差异。 ## 实现方案 ### 方案架构 ``` ┌─────────────────┐ │ Model 1 Eval │ │ + Save Matches │──┐ └─────────────────┘ │ │ ┌──────────────────────┐ ├─→│ Common Match Finder │ │ └──────────────────────┘ ┌─────────────────┐ │ ↓ │ Model 2 Eval │ │ ┌──────────────────────┐ │ + Save Matches │──┘ │ Recompute 3D Stats │ └─────────────────┘ │ (Common Matches Only)│ └──────────────────────┘ ↓ ┌──────────────────────┐ │ Comparison Report │ └──────────────────────┘ ``` ### 数据结构设计 #### 1. 详细匹配数据文件 (`detailed_3d_matches.json`) ```json { "case_019b178d": { "frame_000018": { "vehicle": [ { "gt_id": "hash_of_gt_bbox", // 用于唯一标识GT "gt_bbox": [90.48, 561.98, 241.52, 714.00], "gt_center_3d": [-7.13, 0.91, 8.20], "det_bbox": [91.2, 562.5, 240.8, 713.2], "det_center_3d": [-7.25, 0.89, 8.35], "iou": 0.89, "confidence": 0.87478, "errors": { "lateral": 0.12, "longitudinal": 0.15, "heading": 0.05 }, "distance": { "longitudinal": 8.20, "lateral": -7.13 } } ] } } } ``` #### 2. 共同匹配索引 (`common_matches_index.json`) ```json { "statistics": { "model1_total": 440408, "model2_total": 491768, "common_matches": 425163, "model1_unique": 15245, "model2_unique": 66605, "common_percentage": 96.5 }, "common_matches": { "case_019b178d": { "frame_000018": { "vehicle": [ { "gt_id": "hash_of_gt_bbox", "model1_idx": 0, "model2_idx": 0 } ] } } } } ``` ### 实现步骤 #### 步骤1：扩展评测器保存详细匹配 修改 `eval_tools/evaluator/evaluator.py`： ```python def evaluate_3d(self): # ... 现有代码 ... # 新增：保存详细匹配信息 self.detailed_3d_matches = {} for case_name, case_pairs in cases.items(): self.detailed_3d_matches[case_name] = {} for pair in case_pairs: frame_name = pair['frame'] gts = self.gt_parser.parse_file(...) dets = self.det_parser.parse_file(...) self.detailed_3d_matches[case_name][frame_name] = {} for class_id in Metrics3D.CLASSES_3D: match_result = self.matcher.match(gts, dets, class_id) matches_list = [] for gt_idx, det_idx, iou in match_result['matches']: gt = match_result['gts_filtered'][gt_idx] det = match_result['dets_sorted'][det_idx] if gt['has_3d'] and det['3d_info']: # 计算3D误差 errors = self._compute_3d_errors(gt, det) # 保存匹配详情 matches_list.append({ 'gt_id': self._generate_gt_id(gt), 'gt_bbox': gt['bbox_2d'], 'det_bbox': det['bbox_2d'], 'errors': errors, 'distance': { 'longitudinal': gt['3d_info']['center'][2], 'lateral': gt['3d_info']['center'][0] } }) if matches_list: class_name = self.gt_parser.get_class_name(class_id) self.detailed_3d_matches[case_name][frame_name][class_name] = matches_list ``` #### 步骤2：创建共同匹配查找工具 新建 `eval_tools/find_common_matches.py`： ```python def find_common_matches(model1_matches, model2_matches): """ 找出两个模型共同匹配的GT目标 Args: model1_matches: Model 1的详细匹配数据 model2_matches: Model 2的详细匹配数据 Returns: common_matches: 共同匹配的索引 stats: 统计信息 """ common_matches = {} stats = { 'model1_total': 0, 'model2_total': 0, 'common': 0, 'model1_unique': 0, 'model2_unique': 0 } for case_name in model1_matches: if case_name not in model2_matches: continue common_matches[case_name] = {} for frame_name in model1_matches[case_name]: if frame_name not in model2_matches[case_name]: continue common_matches[case_name][frame_name] = {} for class_name in model1_matches[case_name][frame_name]: if class_name not in model2_matches[case_name][frame_name]: continue m1_list = model1_matches[case_name][frame_name][class_name] m2_list = model2_matches[case_name][frame_name][class_name] # 建立GT ID映射 m1_gt_ids = {m['gt_id']: i for i, m in enumerate(m1_list)} m2_gt_ids = {m['gt_id']: i for i, m in enumerate(m2_list)} # 找共同的GT ID common_gt_ids = set(m1_gt_ids.keys()) & set(m2_gt_ids.keys()) stats['model1_total'] += len(m1_list) stats['model2_total'] += len(m2_list) stats['common'] += len(common_gt_ids) stats['model1_unique'] += len(m1_gt_ids) - len(common_gt_ids) stats['model2_unique'] += len(m2_gt_ids) - len(common_gt_ids) # 保存共同匹配的索引 common_list = [] for gt_id in common_gt_ids: common_list.append({ 'gt_id': gt_id, 'model1_idx': m1_gt_ids[gt_id], 'model2_idx': m2_gt_ids[gt_id] }) if common_list: common_matches[case_name][frame_name][class_name] = common_list return common_matches, stats ``` #### 步骤3：扩展模型比较器 修改 `eval_tools/compare_models.py`： ```python class ModelComparator: def __init__(self, model1_report, model2_report, model1_matches=None, model2_matches=None, use_common_matches=False): """ Args: use_common_matches: 是否只比较共同匹配的样本 """ self.use_common_matches = use_common_matches if use_common_matches: # 找出共同匹配 self.common_matches, self.common_stats = find_common_matches( model1_matches, model2_matches ) # 基于共同匹配重新计算3D统计 self.model1_3d_filtered = self._recompute_3d_stats( model1_matches, 'model1' ) self.model2_3d_filtered = self._recompute_3d_stats( model2_matches, 'model2' ) def _recompute_3d_stats(self, matches_data, model_name): """基于共同匹配重新计算3D统计""" stats = {} for case_name, frames in self.common_matches.items(): for frame_name, classes in frames.items(): for class_name, common_list in classes.items(): if class_name not in stats: stats[class_name] = { 'lateral': [], 'longitudinal': [], 'heading': [] } for match_info in common_list: idx = match_info[f'{model_name}_idx'] match = matches_data[case_name][frame_name][class_name][idx] stats[class_name]['lateral'].append(match['errors']['lateral']) stats[class_name]['longitudinal'].append(match['errors']['longitudinal']) stats[class_name]['heading'].append(match['errors']['heading']) return stats ``` ### 使用流程 #### 1. 评测并保存详细匹配 ```bash # 评测Model 1 python eval_tools/eval.py \ --config eval_tools/configs/eval_config_mono3d.yaml \ --save-detailed-matches # 评测Model 2 python eval_tools/eval.py \ --config eval_tools/configs/eval_config_yolov5.yaml \ --save-detailed-matches ``` #### 2. 比较（基于共同匹配） ```bash python eval_tools/compare_models_with_common_matches.py \ --model1-report eval_results_multiprocess/mono3d/20260203_162537/evaluation_report.json \ --model1-matches eval_results_multiprocess/mono3d/20260203_162537/detailed_3d_matches.json \ --model2-report eval_results_multiprocess/yolov5s/20260203_161644/evaluation_report.json \ --model2-matches eval_results_multiprocess/yolov5s/20260203_161644/detailed_3d_matches.json \ --output-dir comparison_results_common_matches \ --use-common-matches ``` #### 3. 对比报告示例 ``` ================================================================================ 3D METRICS COMPARISON (COMMON MATCHES ONLY) ================================================================================ Match Statistics: Model 1 Total Matches: 440,408 Model 2 Total Matches: 491,768 Common Matches: 425,163 (96.5% of Model 1) Model 1 Unique: 15,245 (3.5%) Model 2 Unique: 66,605 (13.5%) VEHICLE (Common Matches: 420,158): Metric mono3d yolov5s-300w Diff Change % ------------------------------------------------------------------------------------ Lateral (m) 1.3251 1.2103 -0.1148 -8.66% ✓ Longitudinal (m) 2.5892 2.4231 -0.1661 -6.42% ✓ Heading (rad) 0.2256 0.3098 +0.0842 +37.32% ✗ Note: 这些统计基于两个模型都成功匹配的目标，排除了只被一个模型匹配的目标。 ``` ## 优势 1. **公平对比**：确保比较的是相同目标的预测质量 2. **性能分析**：可以单独分析各模型独有匹配的特点 3. **问题诊断**：识别哪些目标一个模型能匹配而另一个不能 ## 可选扩展 ### 1. 分析独有匹配 ```python def analyze_unique_matches(model1_matches, model2_matches, common_matches): """分析每个模型独有匹配的特征""" model1_unique = [] model2_unique = [] # ... 找出独有匹配 ... return { 'model1_unique_analysis': { 'count': len(model1_unique), 'avg_distance': ..., 'avg_iou': ..., 'difficulty': ... # 小目标、遮挡等 }, 'model2_unique_analysis': { ... } } ``` ### 2. 可视化差异 生成可视化图表展示： - 共同匹配 vs 独有匹配的分布 - 不同距离/横向位置的共同匹配率 - 独有匹配的特征分析 ## 实现优先级 1. **P0（必须）**：保存详细匹配信息 2. **P0（必须）**：找出共同匹配并重新计算统计 3. **P1（重要）**：生成基于共同匹配的对比报告 4. **P2（可选）**：分析独有匹配特征 5. **P3（可选）**：可视化工具 ## 兼容性 - **向后兼容**：默认不启用，保持现有行为 - **可选启用**：通过 `--use-common-matches` 标志启用 - **存储开销**：详细匹配文件约为评测报告的 5-10 倍大小