#!/usr/bin/env python3 """Download issue data referenced by the Feishu issue export JSON.""" from __future__ import annotations import argparse import json import re import shutil import subprocess import sys from collections import OrderedDict from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Iterable FILE = Path(__file__).resolve() ROOT = FILE.parents[2] if str(ROOT) not in sys.path: sys.path.append(str(ROOT)) from tools.feishu_project.case_calib_recovery import recover_camera4_json PDCL_REF_RE = re.compile(r"ADAS_[^:/\\\s]+::[^/\\\s]*") MDI_RAW_REF_ARG_RE = re.compile(r"(?:^|\s)mdi\s+raw\b.*?(?:^|\s)-r\s+([^\s]+)") STANDARD_PATH_SPLIT_RE = re.compile(r"[，,\n;；]+") SHARED_CALIB_REL = Path("test_data") / "calibs" / "camera4.json" PLACEHOLDER_TEXTS = {"待填", "待补充", "none", "null", "待提供"} NETWORK_SHARE_PREFIX_MAPPINGS = ( ("//hfs.minieye.tech/project-D4Q2", "/mnt/D4Q2"), ("//192.168.2.122/project-D4Q2", "/mnt/D4Q2"), ("//hfs.minieye.tech/project-G1M3", "/mnt/G1M3"), ("//192.168.2.122/project-G1M3", "/mnt/G1M3"), ("//hfs.minieye.tech/G1M3", "/mnt/G1M3"), ("//192.168.2.122/G1M3", "/mnt/G1M3"), ) @dataclass class ActionResult: issue_id: int issue_name: str source_field: str source_kind: str raw_value: str | None normalized_ref: str | None output_dir: str status: str detail: str resolved_source_path: str | None = None command: list[str] | None = None candidate_paths: list[str] | None = None selected_subpath: str | None = None def to_dict(self) -> dict: return { "issue_id": self.issue_id, "issue_name": self.issue_name, "source_field": self.source_field, "source_kind": self.source_kind, "raw_value": self.raw_value, "normalized_ref": self.normalized_ref, "output_dir": self.output_dir, "status": self.status, "detail": self.detail, "resolved_source_path": self.resolved_source_path, "command": self.command, "candidate_paths": self.candidate_paths, "selected_subpath": self.selected_subpath, } @dataclass(frozen=True) class PDCLDownloadRequest: normalized_ref: str selected_subpath: str | None = None raw_token: str | None = None def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Download or stage issue data from a Feishu issue export JSON." ) parser.add_argument( "--input-json", default="tools/feishu_project/dongying_g1q3_issue_list.json", help="Path to the issue export JSON.", ) parser.add_argument( "--output-root", required=True, help="Directory where downloaded or copied data should be stored.", ) parser.add_argument( "--manifest-path", default=None, help="Optional explicit path for the execution manifest JSON.", ) parser.add_argument( "--issue-id", action="append", dest="issue_ids", type=int, help="Optional issue id filter. Can be repeated.", ) parser.add_argument( "--dry-run", action="store_true", help="Plan actions without running mdi or copying files.", ) parser.add_argument( "--skip-mdi", action="store_true", help="Skip PDCL/MDI downloads and only process standard paths.", ) parser.add_argument( "--skip-copy", action="store_true", help="Skip standard-path copies and only process PDCL/MDI downloads.", ) parser.add_argument( "--only-redownload-affected-cases", action="store_true", help=( "Only re-copy standard-path cases affected by the historical sigmastar.1/camera4.bin " "copy bug. This mode skips PDCL/MDI downloads and replaces stale copied targets." ), ) parser.add_argument( "--skip-calib-recovery", action="store_true", help=( "Skip recovering camera4.json from camera_config_folder.bin or mcap attachments " "after standard-path copies." ), ) return parser.parse_args() def load_issue_items(path: Path) -> list[dict]: payload = json.loads(path.read_text(encoding="utf-8")) return payload["items"] def ensure_dir(path: Path, dry_run: bool) -> None: if dry_run: return path.mkdir(parents=True, exist_ok=True) def log_progress(message: str) -> None: timestamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S") print(f"[download_issue_data {timestamp}] {message}", flush=True) def compact_text(value: object, max_len: int = 96) -> str: text = "" if value is None else str(value).strip() text = re.sub(r"\s+", " ", text) if len(text) <= max_len: return text return f"{text[: max_len - 3]}..." def summarize_issue_results(results: list[ActionResult]) -> str: if not results: return "no actions" summary: dict[str, int] = {} for result in results: summary[result.status] = summary.get(result.status, 0) + 1 return ", ".join(f"{status}={summary[status]}" for status in sorted(summary)) def normalize_issue_dirname(issue_id: int) -> str: return f"issue_{issue_id}" def iter_issue_fields(item: dict) -> Iterable[tuple[str, object]]: yield "问题数据地址", item.get("问题数据地址") yield "问题数据地址_PDCL", item.get("问题数据地址_PDCL") def _normalize_pdcl_selected_subpath(raw_subpath: str) -> str | None: cleaned = raw_subpath.strip().strip("/") if not cleaned: return None candidate = Path(cleaned) if candidate.is_absolute(): return None if any(part in {"", ".", ".."} for part in candidate.parts): return None return str(candidate) def _build_pdcl_request_from_token(token: str) -> PDCLDownloadRequest | None: stripped = token.strip().strip("\"'`") match = PDCL_REF_RE.match(stripped) if match is None: return None normalized_ref = match.group(0) suffix = stripped[match.end():] selected_subpath = None if suffix.startswith("/"): raw_subpath = suffix[1:] if raw_subpath and not raw_subpath.startswith("ADAS_"): selected_subpath = _normalize_pdcl_selected_subpath(raw_subpath) return PDCLDownloadRequest( normalized_ref=normalized_ref, selected_subpath=selected_subpath, raw_token=stripped, ) def extract_pdcl_requests(raw_value: object) -> list[PDCLDownloadRequest]: if raw_value is None: return [] text = str(raw_value).strip() if not text: return [] requests: list[PDCLDownloadRequest] = [] for segment in (part.strip() for part in STANDARD_PATH_SPLIT_RE.split(text)): if not segment: continue mdi_match = MDI_RAW_REF_ARG_RE.search(segment) if mdi_match is not None: request = _build_pdcl_request_from_token(mdi_match.group(1)) if request is not None: requests.append(request) continue search_pos = 0 while True: match = PDCL_REF_RE.search(segment, search_pos) if match is None: break normalized_ref = match.group(0) suffix = segment[match.end():] selected_subpath = None if suffix.startswith("/") and not suffix[1:].startswith("ADAS_"): selected_subpath = _normalize_pdcl_selected_subpath(suffix[1:]) requests.append( PDCLDownloadRequest( normalized_ref=normalized_ref, selected_subpath=selected_subpath, raw_token=segment, ) ) break requests.append( PDCLDownloadRequest( normalized_ref=normalized_ref, selected_subpath=None, raw_token=normalized_ref, ) ) search_pos = match.end() deduped: list[PDCLDownloadRequest] = [] seen = set() for request in requests: key = (request.normalized_ref, request.selected_subpath) if key in seen: continue seen.add(key) deduped.append(request) return deduped def extract_standard_paths(raw_value: object) -> list[str]: if raw_value is None: return [] text = str(raw_value).strip() if not text: return [] if text.lower() in PLACEHOLDER_TEXTS: return [] if extract_pdcl_requests(text): return [] if "/" not in text and "\\" not in text: return [] parts = [part.strip() for part in STANDARD_PATH_SPLIT_RE.split(text)] return [part for part in parts if part] def normalize_standard_source_path(path: Path) -> Path: normalized = path if normalized.name == "camera4.bin" and normalized.parent.name == "sigmastar.1": return normalized.parent.parent if normalized.name == "sigmastar.1": return normalized.parent return normalized def is_affected_standard_path(raw_path: str) -> bool: raw_path_obj = Path(raw_path.strip()) return normalize_standard_source_path(raw_path_obj) != raw_path_obj def normalize_share_path_separators(path_str: str) -> str: normalized = path_str.strip().replace("\\", "/") normalized = re.sub(r"/{3,}", "//", normalized) return normalized def rewrite_network_share_path(path_str: str) -> str | None: normalized = normalize_share_path_separators(path_str) for prefix_src, prefix_dst in NETWORK_SHARE_PREFIX_MAPPINGS: if normalized.startswith(prefix_src): return f"{prefix_dst}{normalized[len(prefix_src):]}" return None def build_path_candidates(raw_path: str) -> list[Path]: candidates: list[str] = [raw_path] normalized_share = normalize_share_path_separators(raw_path) if normalized_share != raw_path: candidates.append(normalized_share) network_share_rewritten = rewrite_network_share_path(raw_path) if network_share_rewritten: candidates.append(network_share_rewritten) for needle in ("hfs/project-G1M3", "project-G1M3"): for candidate in list(candidates): if needle in candidate: candidates.append(candidate.replace(needle, "G1M3")) normalized_candidates = [normalize_standard_source_path(Path(candidate)) for candidate in candidates] unique_candidates = list(OrderedDict.fromkeys(str(candidate) for candidate in normalized_candidates)) return [Path(candidate) for candidate in unique_candidates] def resolve_existing_path(raw_path: str) -> tuple[Path | None, list[Path]]: candidates = build_path_candidates(raw_path) for candidate in candidates: if candidate.exists(): return candidate, candidates return None, candidates def remove_existing_target(path: Path) -> None: if path.is_dir(): shutil.rmtree(path) else: path.unlink() def copy_source_path( source_path: Path, output_dir: Path, dry_run: bool, replace_existing: bool = False, legacy_target_names: Iterable[str] | None = None, ) -> tuple[str, str]: target = output_dir / source_path.name cleanup_targets: list[Path] = [target] if legacy_target_names: for target_name in legacy_target_names: cleanup_targets.append(output_dir / target_name) unique_cleanup_targets = list(OrderedDict.fromkeys(str(path) for path in cleanup_targets)) cleanup_paths = [Path(path) for path in unique_cleanup_targets] existing_targets = [path for path in cleanup_paths if path.exists()] if existing_targets and not replace_existing: return "exists", f"target already exists: {target}" if dry_run: if existing_targets and replace_existing: existing_str = ", ".join(str(path) for path in existing_targets) return "planned_redownload", f"would replace {existing_str} with {source_path} -> {target}" return "planned", f"would copy {source_path} -> {target}" ensure_dir(output_dir, dry_run=False) if replace_existing: for existing_target in existing_targets: remove_existing_target(existing_target) if source_path.is_dir(): shutil.copytree(source_path, target) else: shutil.copy2(source_path, target) if existing_targets and replace_existing: replaced = ", ".join(str(path) for path in existing_targets) return "redownloaded", f"replaced {replaced} with {target}" return "copied", f"copied to {target}" def build_copied_target_root(output_dir: Path, source_path: Path) -> Path: return output_dir / source_path.name def find_shared_test_data_dir(source_path: Path, max_parent_levels: int = 4) -> Path | None: if (source_path / SHARED_CALIB_REL).is_file(): return None current = source_path.parent for _ in range(max_parent_levels): candidate = current / "test_data" if (candidate / "calibs" / "camera4.json").is_file(): return candidate if current.parent == current: break current = current.parent return None def sync_shared_test_data( source_path: Path, target_root: Path, dry_run: bool, ) -> tuple[str | None, str | None, str | None]: shared_test_data_dir = find_shared_test_data_dir(source_path) if shared_test_data_dir is None: return None, None, None target_shared_test_data_dir = target_root / "test_data" target_shared_calib = target_shared_test_data_dir / "calibs" / "camera4.json" if target_shared_calib.is_file(): return None, None, None if dry_run: return ( "planned_shared_calib_sync", f"would copy shared test_data {shared_test_data_dir} -> {target_shared_test_data_dir}", str(shared_test_data_dir), ) shutil.copytree(shared_test_data_dir, target_shared_test_data_dir, dirs_exist_ok=True) return ( "synced_shared_calib", f"copied shared test_data {shared_test_data_dir} -> {target_shared_test_data_dir}", str(shared_test_data_dir), ) def recover_target_root_calib( source_root: Path, target_root: Path, dry_run: bool, ) -> tuple[str, str, str | None]: recovery = recover_camera4_json( source_root=source_root, target_root=target_root, dry_run=dry_run, ) return recovery.status, recovery.detail, None if recovery.source_path is None else str(recovery.source_path) def _expected_pdcl_root_dir(output_dir: Path, ref: str) -> Path: if "::" not in ref: raise ValueError(f"Unexpected PDCL ref without '::': {ref}") return output_dir / ref.split("::", 1)[1] def _prune_pdcl_download_to_selected_subpath( root_dir: Path, selected_subpath: str, dry_run: bool, ) -> tuple[str, str]: selected_rel = Path(selected_subpath) selected_path = root_dir / selected_rel if dry_run: return ( "planned_selected_subpath", f"would keep {selected_path} and shared test_data under {root_dir}", ) if not selected_path.exists(): return ( "failed_selected_subpath_missing", f"selected subpath not found after mdi download: {selected_path}", ) keep_names = {selected_rel.parts[0], "test_data"} removed_children: list[str] = [] for child in root_dir.iterdir(): if child.name in keep_names: continue removed_children.append(child.name) remove_existing_target(child) detail = f"kept selected subpath {selected_path}" if removed_children: detail += f"; removed siblings: {', '.join(sorted(removed_children))}" return "downloaded_selected_subpath", detail def run_mdi_download(request: PDCLDownloadRequest, output_dir: Path, dry_run: bool) -> tuple[str, str, list[str]]: command = ["mdi", "raw", "-r", request.normalized_ref, "-s", str(output_dir)] if dry_run: if request.selected_subpath: root_dir = _expected_pdcl_root_dir(output_dir, request.normalized_ref) status, detail = _prune_pdcl_download_to_selected_subpath(root_dir, request.selected_subpath, dry_run=True) return status, f"would run {' '.join(command)}; {detail}", command return "planned", f"would run {' '.join(command)}", command ensure_dir(output_dir, dry_run=False) completed = subprocess.run( command, check=False, capture_output=True, text=True, encoding="utf-8", ) if completed.returncode == 0: base_detail = completed.stdout.strip() or "mdi raw completed" if request.selected_subpath: root_dir = _expected_pdcl_root_dir(output_dir, request.normalized_ref) status, detail = _prune_pdcl_download_to_selected_subpath(root_dir, request.selected_subpath, dry_run=False) return status, f"{base_detail}\n{detail}", command return "downloaded", base_detail, command detail = completed.stderr.strip() or completed.stdout.strip() or "mdi raw failed" return "failed", detail, command def process_issue( item: dict, output_root: Path, dry_run: bool, skip_mdi: bool, skip_copy: bool, only_redownload_affected_cases: bool, skip_calib_recovery: bool, ) -> list[ActionResult]: issue_id = int(item["id"]) issue_name = str(item["name"]) issue_dir = output_root / normalize_issue_dirname(issue_id) results: list[ActionResult] = [] seen_pdcl_requests: set[tuple[str, str | None]] = set() seen_paths: set[str] = set() pdcl_index = 0 path_index = 0 for field_name, raw_value in iter_issue_fields(item): if not skip_mdi and not only_redownload_affected_cases: for request in extract_pdcl_requests(raw_value): request_key = (request.normalized_ref, request.selected_subpath) if request_key in seen_pdcl_requests: results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="pdcl_mdi_download", raw_value=None if raw_value is None else str(raw_value), normalized_ref=request.normalized_ref, output_dir=str(issue_dir), status="skipped_duplicate", detail=( f"duplicate PDCL ref: {request.normalized_ref}" if request.selected_subpath is None else f"duplicate PDCL ref+subpath: {request.normalized_ref} / {request.selected_subpath}" ), selected_subpath=request.selected_subpath, ) ) continue seen_pdcl_requests.add(request_key) pdcl_index += 1 download_dir = issue_dir / f"pdcl_{pdcl_index:02d}" request_desc = request.normalized_ref if request.selected_subpath: request_desc += f"/{request.selected_subpath}" log_progress( f"issue_{issue_id} [download] pdcl_{pdcl_index:02d}: {compact_text(request_desc)}" ) status, detail, command = run_mdi_download(request, download_dir, dry_run=dry_run) results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="pdcl_mdi_download", raw_value=None if raw_value is None else str(raw_value), normalized_ref=request.normalized_ref, output_dir=str(download_dir), status=status, detail=detail, command=command, selected_subpath=request.selected_subpath, ) ) if skip_copy: continue for raw_path in extract_standard_paths(raw_value): if raw_path in seen_paths: results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="standard_path", raw_value=raw_path, normalized_ref=None, output_dir=str(issue_dir), status="skipped_duplicate", detail=f"duplicate standard path: {raw_path}", ) ) continue seen_paths.add(raw_path) path_index += 1 copy_dir = issue_dir / f"path_{path_index:02d}" affected_standard_path = is_affected_standard_path(raw_path) if only_redownload_affected_cases and not affected_standard_path: continue log_progress( f"issue_{issue_id} [download] path_{path_index:02d}: {compact_text(raw_path)}" ) resolved_source_path, candidates = resolve_existing_path(raw_path) if resolved_source_path is None: results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="standard_path", raw_value=raw_path, normalized_ref=None, output_dir=str(copy_dir), status="skipped_missing", detail="source path not found after rewrite attempts", candidate_paths=[str(candidate) for candidate in candidates], ) ) continue legacy_target_names = [] if affected_standard_path: legacy_target_names.append(Path(raw_path.strip()).name) status, detail = copy_source_path( resolved_source_path, copy_dir, dry_run=dry_run, replace_existing=only_redownload_affected_cases and affected_standard_path, legacy_target_names=legacy_target_names, ) target_root = build_copied_target_root(copy_dir, resolved_source_path) results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="standard_path", raw_value=raw_path, normalized_ref=None, output_dir=str(copy_dir), status=status, detail=detail, resolved_source_path=str(resolved_source_path), candidate_paths=[str(candidate) for candidate in candidates], ) ) sync_status, sync_detail, shared_source_dir = sync_shared_test_data( resolved_source_path, target_root, dry_run=dry_run, ) if sync_status is not None: results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="shared_test_data", raw_value=raw_path, normalized_ref=None, output_dir=str(target_root / "test_data"), status=sync_status, detail=sync_detail or "", resolved_source_path=shared_source_dir, candidate_paths=[str(candidate) for candidate in candidates], ) ) if not skip_calib_recovery: calib_status, calib_detail, calib_source_path = recover_target_root_calib( source_root=resolved_source_path, target_root=target_root, dry_run=dry_run, ) results.append( ActionResult( issue_id=issue_id, issue_name=issue_name, source_field=field_name, source_kind="case_calib_recovery", raw_value=raw_path, normalized_ref=None, output_dir=str(target_root / "test_data" / "calibs"), status=calib_status, detail=calib_detail, resolved_source_path=calib_source_path, candidate_paths=[str(candidate) for candidate in candidates], ) ) return results def build_manifest( args: argparse.Namespace, input_json: Path, output_root: Path, action_results: list[ActionResult], ) -> dict: summary: dict[str, int] = {} for action in action_results: summary[action.status] = summary.get(action.status, 0) + 1 return { "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"), "input_json": str(input_json), "output_root": str(output_root), "dry_run": args.dry_run, "skip_mdi": args.skip_mdi, "skip_copy": args.skip_copy, "skip_calib_recovery": args.skip_calib_recovery, "only_redownload_affected_cases": args.only_redownload_affected_cases, "issue_filter": args.issue_ids or [], "summary": summary, "actions": [action.to_dict() for action in action_results], } def print_summary(manifest: dict) -> None: print(f"input_json: {manifest['input_json']}") print(f"output_root: {manifest['output_root']}") print(f"dry_run: {manifest['dry_run']}") for status, count in sorted(manifest["summary"].items()): print(f"{status}: {count}") def main() -> int: args = parse_args() input_json = Path(args.input_json).resolve() output_root = Path(args.output_root).resolve() manifest_path = ( Path(args.manifest_path).resolve() if args.manifest_path else output_root / "download_manifest.json" ) items = load_issue_items(input_json) if args.issue_ids: issue_filter = set(args.issue_ids) items = [item for item in items if int(item["id"]) in issue_filter] ensure_dir(output_root, dry_run=args.dry_run) action_results: list[ActionResult] = [] total_items = len(items) log_progress(f"issues_to_process: {total_items}") for index, item in enumerate(items, start=1): issue_id = int(item["id"]) issue_name = compact_text(item.get("name"), max_len=64) log_progress(f"[{index}/{total_items}] issue_{issue_id} start: {issue_name}") issue_results = process_issue( item=item, output_root=output_root, dry_run=args.dry_run, skip_mdi=args.skip_mdi, skip_copy=args.skip_copy, only_redownload_affected_cases=args.only_redownload_affected_cases, skip_calib_recovery=args.skip_calib_recovery, ) action_results.extend(issue_results) log_progress( f"[{index}/{total_items}] issue_{issue_id} done: {summarize_issue_results(issue_results)}" ) manifest = build_manifest(args, input_json, output_root, action_results) ensure_dir(manifest_path.parent, dry_run=args.dry_run) if not args.dry_run: manifest_path.write_text( json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print_summary(manifest) if args.dry_run: print(f"manifest (not written in dry-run): {manifest_path}") else: print(f"manifest: {manifest_path}") return 0 if __name__ == "__main__": sys.exit(main())