806 lines
28 KiB
Python
Executable File
806 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Download issue data referenced by the Feishu issue export JSON."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
from collections import OrderedDict
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Iterable
|
||
|
||
FILE = Path(__file__).resolve()
|
||
ROOT = FILE.parents[2]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.append(str(ROOT))
|
||
|
||
from tools.feishu_project.case_calib_recovery import recover_camera4_json
|
||
|
||
|
||
PDCL_REF_RE = re.compile(r"ADAS_[^:/\\\s]+::[^/\\\s]*")
|
||
MDI_RAW_REF_ARG_RE = re.compile(r"(?:^|\s)mdi\s+raw\b.*?(?:^|\s)-r\s+([^\s]+)")
|
||
STANDARD_PATH_SPLIT_RE = re.compile(r"[，,\n;；]+")
|
||
SHARED_CALIB_REL = Path("test_data") / "calibs" / "camera4.json"
|
||
PLACEHOLDER_TEXTS = {"待填", "待补充", "none", "null", "待提供"}
|
||
NETWORK_SHARE_PREFIX_MAPPINGS = (
|
||
("//hfs.minieye.tech/project-D4Q2", "/mnt/D4Q2"),
|
||
("//192.168.2.122/project-D4Q2", "/mnt/D4Q2"),
|
||
("//hfs.minieye.tech/project-G1M3", "/mnt/G1M3"),
|
||
("//192.168.2.122/project-G1M3", "/mnt/G1M3"),
|
||
("//hfs.minieye.tech/G1M3", "/mnt/G1M3"),
|
||
("//192.168.2.122/G1M3", "/mnt/G1M3"),
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class ActionResult:
|
||
issue_id: int
|
||
issue_name: str
|
||
source_field: str
|
||
source_kind: str
|
||
raw_value: str | None
|
||
normalized_ref: str | None
|
||
output_dir: str
|
||
status: str
|
||
detail: str
|
||
resolved_source_path: str | None = None
|
||
command: list[str] | None = None
|
||
candidate_paths: list[str] | None = None
|
||
selected_subpath: str | None = None
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"issue_id": self.issue_id,
|
||
"issue_name": self.issue_name,
|
||
"source_field": self.source_field,
|
||
"source_kind": self.source_kind,
|
||
"raw_value": self.raw_value,
|
||
"normalized_ref": self.normalized_ref,
|
||
"output_dir": self.output_dir,
|
||
"status": self.status,
|
||
"detail": self.detail,
|
||
"resolved_source_path": self.resolved_source_path,
|
||
"command": self.command,
|
||
"candidate_paths": self.candidate_paths,
|
||
"selected_subpath": self.selected_subpath,
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class PDCLDownloadRequest:
|
||
normalized_ref: str
|
||
selected_subpath: str | None = None
|
||
raw_token: str | None = None
|
||
|
||
|
||
def parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description="Download or stage issue data from a Feishu issue export JSON."
|
||
)
|
||
parser.add_argument(
|
||
"--input-json",
|
||
default="tools/feishu_project/dongying_g1q3_issue_list.json",
|
||
help="Path to the issue export JSON.",
|
||
)
|
||
parser.add_argument(
|
||
"--output-root",
|
||
required=True,
|
||
help="Directory where downloaded or copied data should be stored.",
|
||
)
|
||
parser.add_argument(
|
||
"--manifest-path",
|
||
default=None,
|
||
help="Optional explicit path for the execution manifest JSON.",
|
||
)
|
||
parser.add_argument(
|
||
"--issue-id",
|
||
action="append",
|
||
dest="issue_ids",
|
||
type=int,
|
||
help="Optional issue id filter. Can be repeated.",
|
||
)
|
||
parser.add_argument(
|
||
"--dry-run",
|
||
action="store_true",
|
||
help="Plan actions without running mdi or copying files.",
|
||
)
|
||
parser.add_argument(
|
||
"--skip-mdi",
|
||
action="store_true",
|
||
help="Skip PDCL/MDI downloads and only process standard paths.",
|
||
)
|
||
parser.add_argument(
|
||
"--skip-copy",
|
||
action="store_true",
|
||
help="Skip standard-path copies and only process PDCL/MDI downloads.",
|
||
)
|
||
parser.add_argument(
|
||
"--only-redownload-affected-cases",
|
||
action="store_true",
|
||
help=(
|
||
"Only re-copy standard-path cases affected by the historical sigmastar.1/camera4.bin "
|
||
"copy bug. This mode skips PDCL/MDI downloads and replaces stale copied targets."
|
||
),
|
||
)
|
||
parser.add_argument(
|
||
"--skip-calib-recovery",
|
||
action="store_true",
|
||
help=(
|
||
"Skip recovering camera4.json from camera_config_folder.bin or mcap attachments "
|
||
"after standard-path copies."
|
||
),
|
||
)
|
||
return parser.parse_args()
|
||
|
||
|
||
def load_issue_items(path: Path) -> list[dict]:
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
return payload["items"]
|
||
|
||
|
||
def ensure_dir(path: Path, dry_run: bool) -> None:
|
||
if dry_run:
|
||
return
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
|
||
|
||
def log_progress(message: str) -> None:
|
||
timestamp = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S")
|
||
print(f"[download_issue_data {timestamp}] {message}", flush=True)
|
||
|
||
|
||
def compact_text(value: object, max_len: int = 96) -> str:
|
||
text = "" if value is None else str(value).strip()
|
||
text = re.sub(r"\s+", " ", text)
|
||
if len(text) <= max_len:
|
||
return text
|
||
return f"{text[: max_len - 3]}..."
|
||
|
||
|
||
def summarize_issue_results(results: list[ActionResult]) -> str:
|
||
if not results:
|
||
return "no actions"
|
||
summary: dict[str, int] = {}
|
||
for result in results:
|
||
summary[result.status] = summary.get(result.status, 0) + 1
|
||
return ", ".join(f"{status}={summary[status]}" for status in sorted(summary))
|
||
|
||
|
||
def normalize_issue_dirname(issue_id: int) -> str:
|
||
return f"issue_{issue_id}"
|
||
|
||
|
||
def iter_issue_fields(item: dict) -> Iterable[tuple[str, object]]:
|
||
yield "问题数据地址", item.get("问题数据地址")
|
||
yield "问题数据地址_PDCL", item.get("问题数据地址_PDCL")
|
||
|
||
|
||
def _normalize_pdcl_selected_subpath(raw_subpath: str) -> str | None:
|
||
cleaned = raw_subpath.strip().strip("/")
|
||
if not cleaned:
|
||
return None
|
||
candidate = Path(cleaned)
|
||
if candidate.is_absolute():
|
||
return None
|
||
if any(part in {"", ".", ".."} for part in candidate.parts):
|
||
return None
|
||
return str(candidate)
|
||
|
||
|
||
def _build_pdcl_request_from_token(token: str) -> PDCLDownloadRequest | None:
|
||
stripped = token.strip().strip("\"'`")
|
||
match = PDCL_REF_RE.match(stripped)
|
||
if match is None:
|
||
return None
|
||
normalized_ref = match.group(0)
|
||
suffix = stripped[match.end():]
|
||
selected_subpath = None
|
||
if suffix.startswith("/"):
|
||
raw_subpath = suffix[1:]
|
||
if raw_subpath and not raw_subpath.startswith("ADAS_"):
|
||
selected_subpath = _normalize_pdcl_selected_subpath(raw_subpath)
|
||
return PDCLDownloadRequest(
|
||
normalized_ref=normalized_ref,
|
||
selected_subpath=selected_subpath,
|
||
raw_token=stripped,
|
||
)
|
||
|
||
|
||
def extract_pdcl_requests(raw_value: object) -> list[PDCLDownloadRequest]:
|
||
if raw_value is None:
|
||
return []
|
||
text = str(raw_value).strip()
|
||
if not text:
|
||
return []
|
||
requests: list[PDCLDownloadRequest] = []
|
||
|
||
for segment in (part.strip() for part in STANDARD_PATH_SPLIT_RE.split(text)):
|
||
if not segment:
|
||
continue
|
||
|
||
mdi_match = MDI_RAW_REF_ARG_RE.search(segment)
|
||
if mdi_match is not None:
|
||
request = _build_pdcl_request_from_token(mdi_match.group(1))
|
||
if request is not None:
|
||
requests.append(request)
|
||
continue
|
||
|
||
search_pos = 0
|
||
while True:
|
||
match = PDCL_REF_RE.search(segment, search_pos)
|
||
if match is None:
|
||
break
|
||
normalized_ref = match.group(0)
|
||
suffix = segment[match.end():]
|
||
selected_subpath = None
|
||
if suffix.startswith("/") and not suffix[1:].startswith("ADAS_"):
|
||
selected_subpath = _normalize_pdcl_selected_subpath(suffix[1:])
|
||
requests.append(
|
||
PDCLDownloadRequest(
|
||
normalized_ref=normalized_ref,
|
||
selected_subpath=selected_subpath,
|
||
raw_token=segment,
|
||
)
|
||
)
|
||
break
|
||
|
||
requests.append(
|
||
PDCLDownloadRequest(
|
||
normalized_ref=normalized_ref,
|
||
selected_subpath=None,
|
||
raw_token=normalized_ref,
|
||
)
|
||
)
|
||
search_pos = match.end()
|
||
|
||
deduped: list[PDCLDownloadRequest] = []
|
||
seen = set()
|
||
for request in requests:
|
||
key = (request.normalized_ref, request.selected_subpath)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
deduped.append(request)
|
||
return deduped
|
||
|
||
|
||
def extract_standard_paths(raw_value: object) -> list[str]:
|
||
if raw_value is None:
|
||
return []
|
||
text = str(raw_value).strip()
|
||
if not text:
|
||
return []
|
||
if text.lower() in PLACEHOLDER_TEXTS:
|
||
return []
|
||
if extract_pdcl_requests(text):
|
||
return []
|
||
if "/" not in text and "\\" not in text:
|
||
return []
|
||
parts = [part.strip() for part in STANDARD_PATH_SPLIT_RE.split(text)]
|
||
return [part for part in parts if part]
|
||
|
||
|
||
def normalize_standard_source_path(path: Path) -> Path:
|
||
normalized = path
|
||
if normalized.name == "camera4.bin" and normalized.parent.name == "sigmastar.1":
|
||
return normalized.parent.parent
|
||
if normalized.name == "sigmastar.1":
|
||
return normalized.parent
|
||
return normalized
|
||
|
||
|
||
def is_affected_standard_path(raw_path: str) -> bool:
|
||
raw_path_obj = Path(raw_path.strip())
|
||
return normalize_standard_source_path(raw_path_obj) != raw_path_obj
|
||
|
||
|
||
def normalize_share_path_separators(path_str: str) -> str:
|
||
normalized = path_str.strip().replace("\\", "/")
|
||
normalized = re.sub(r"/{3,}", "//", normalized)
|
||
return normalized
|
||
|
||
|
||
def rewrite_network_share_path(path_str: str) -> str | None:
|
||
normalized = normalize_share_path_separators(path_str)
|
||
for prefix_src, prefix_dst in NETWORK_SHARE_PREFIX_MAPPINGS:
|
||
if normalized.startswith(prefix_src):
|
||
return f"{prefix_dst}{normalized[len(prefix_src):]}"
|
||
return None
|
||
|
||
|
||
def build_path_candidates(raw_path: str) -> list[Path]:
|
||
candidates: list[str] = [raw_path]
|
||
|
||
normalized_share = normalize_share_path_separators(raw_path)
|
||
if normalized_share != raw_path:
|
||
candidates.append(normalized_share)
|
||
|
||
network_share_rewritten = rewrite_network_share_path(raw_path)
|
||
if network_share_rewritten:
|
||
candidates.append(network_share_rewritten)
|
||
|
||
for needle in ("hfs/project-G1M3", "project-G1M3"):
|
||
for candidate in list(candidates):
|
||
if needle in candidate:
|
||
candidates.append(candidate.replace(needle, "G1M3"))
|
||
|
||
normalized_candidates = [normalize_standard_source_path(Path(candidate)) for candidate in candidates]
|
||
unique_candidates = list(OrderedDict.fromkeys(str(candidate) for candidate in normalized_candidates))
|
||
return [Path(candidate) for candidate in unique_candidates]
|
||
|
||
|
||
def resolve_existing_path(raw_path: str) -> tuple[Path | None, list[Path]]:
|
||
candidates = build_path_candidates(raw_path)
|
||
for candidate in candidates:
|
||
if candidate.exists():
|
||
return candidate, candidates
|
||
return None, candidates
|
||
|
||
|
||
def remove_existing_target(path: Path) -> None:
|
||
if path.is_dir():
|
||
shutil.rmtree(path)
|
||
else:
|
||
path.unlink()
|
||
|
||
|
||
def copy_source_path(
|
||
source_path: Path,
|
||
output_dir: Path,
|
||
dry_run: bool,
|
||
replace_existing: bool = False,
|
||
legacy_target_names: Iterable[str] | None = None,
|
||
) -> tuple[str, str]:
|
||
target = output_dir / source_path.name
|
||
cleanup_targets: list[Path] = [target]
|
||
if legacy_target_names:
|
||
for target_name in legacy_target_names:
|
||
cleanup_targets.append(output_dir / target_name)
|
||
|
||
unique_cleanup_targets = list(OrderedDict.fromkeys(str(path) for path in cleanup_targets))
|
||
cleanup_paths = [Path(path) for path in unique_cleanup_targets]
|
||
existing_targets = [path for path in cleanup_paths if path.exists()]
|
||
|
||
if existing_targets and not replace_existing:
|
||
return "exists", f"target already exists: {target}"
|
||
|
||
if dry_run:
|
||
if existing_targets and replace_existing:
|
||
existing_str = ", ".join(str(path) for path in existing_targets)
|
||
return "planned_redownload", f"would replace {existing_str} with {source_path} -> {target}"
|
||
return "planned", f"would copy {source_path} -> {target}"
|
||
|
||
ensure_dir(output_dir, dry_run=False)
|
||
if replace_existing:
|
||
for existing_target in existing_targets:
|
||
remove_existing_target(existing_target)
|
||
|
||
if source_path.is_dir():
|
||
shutil.copytree(source_path, target)
|
||
else:
|
||
shutil.copy2(source_path, target)
|
||
if existing_targets and replace_existing:
|
||
replaced = ", ".join(str(path) for path in existing_targets)
|
||
return "redownloaded", f"replaced {replaced} with {target}"
|
||
return "copied", f"copied to {target}"
|
||
|
||
|
||
def build_copied_target_root(output_dir: Path, source_path: Path) -> Path:
|
||
return output_dir / source_path.name
|
||
|
||
|
||
def find_shared_test_data_dir(source_path: Path, max_parent_levels: int = 4) -> Path | None:
|
||
if (source_path / SHARED_CALIB_REL).is_file():
|
||
return None
|
||
|
||
current = source_path.parent
|
||
for _ in range(max_parent_levels):
|
||
candidate = current / "test_data"
|
||
if (candidate / "calibs" / "camera4.json").is_file():
|
||
return candidate
|
||
if current.parent == current:
|
||
break
|
||
current = current.parent
|
||
return None
|
||
|
||
|
||
def sync_shared_test_data(
|
||
source_path: Path,
|
||
target_root: Path,
|
||
dry_run: bool,
|
||
) -> tuple[str | None, str | None, str | None]:
|
||
shared_test_data_dir = find_shared_test_data_dir(source_path)
|
||
if shared_test_data_dir is None:
|
||
return None, None, None
|
||
|
||
target_shared_test_data_dir = target_root / "test_data"
|
||
target_shared_calib = target_shared_test_data_dir / "calibs" / "camera4.json"
|
||
if target_shared_calib.is_file():
|
||
return None, None, None
|
||
|
||
if dry_run:
|
||
return (
|
||
"planned_shared_calib_sync",
|
||
f"would copy shared test_data {shared_test_data_dir} -> {target_shared_test_data_dir}",
|
||
str(shared_test_data_dir),
|
||
)
|
||
|
||
shutil.copytree(shared_test_data_dir, target_shared_test_data_dir, dirs_exist_ok=True)
|
||
return (
|
||
"synced_shared_calib",
|
||
f"copied shared test_data {shared_test_data_dir} -> {target_shared_test_data_dir}",
|
||
str(shared_test_data_dir),
|
||
)
|
||
|
||
|
||
def recover_target_root_calib(
|
||
source_root: Path,
|
||
target_root: Path,
|
||
dry_run: bool,
|
||
) -> tuple[str, str, str | None]:
|
||
recovery = recover_camera4_json(
|
||
source_root=source_root,
|
||
target_root=target_root,
|
||
dry_run=dry_run,
|
||
)
|
||
return recovery.status, recovery.detail, None if recovery.source_path is None else str(recovery.source_path)
|
||
|
||
|
||
def _expected_pdcl_root_dir(output_dir: Path, ref: str) -> Path:
|
||
if "::" not in ref:
|
||
raise ValueError(f"Unexpected PDCL ref without '::': {ref}")
|
||
return output_dir / ref.split("::", 1)[1]
|
||
|
||
|
||
def _prune_pdcl_download_to_selected_subpath(
|
||
root_dir: Path,
|
||
selected_subpath: str,
|
||
dry_run: bool,
|
||
) -> tuple[str, str]:
|
||
selected_rel = Path(selected_subpath)
|
||
selected_path = root_dir / selected_rel
|
||
if dry_run:
|
||
return (
|
||
"planned_selected_subpath",
|
||
f"would keep {selected_path} and shared test_data under {root_dir}",
|
||
)
|
||
|
||
if not selected_path.exists():
|
||
return (
|
||
"failed_selected_subpath_missing",
|
||
f"selected subpath not found after mdi download: {selected_path}",
|
||
)
|
||
|
||
keep_names = {selected_rel.parts[0], "test_data"}
|
||
removed_children: list[str] = []
|
||
|
||
for child in root_dir.iterdir():
|
||
if child.name in keep_names:
|
||
continue
|
||
removed_children.append(child.name)
|
||
remove_existing_target(child)
|
||
|
||
detail = f"kept selected subpath {selected_path}"
|
||
if removed_children:
|
||
detail += f"; removed siblings: {', '.join(sorted(removed_children))}"
|
||
return "downloaded_selected_subpath", detail
|
||
|
||
|
||
def run_mdi_download(request: PDCLDownloadRequest, output_dir: Path, dry_run: bool) -> tuple[str, str, list[str]]:
|
||
command = ["mdi", "raw", "-r", request.normalized_ref, "-s", str(output_dir)]
|
||
if dry_run:
|
||
if request.selected_subpath:
|
||
root_dir = _expected_pdcl_root_dir(output_dir, request.normalized_ref)
|
||
status, detail = _prune_pdcl_download_to_selected_subpath(root_dir, request.selected_subpath, dry_run=True)
|
||
return status, f"would run {' '.join(command)}; {detail}", command
|
||
return "planned", f"would run {' '.join(command)}", command
|
||
|
||
ensure_dir(output_dir, dry_run=False)
|
||
completed = subprocess.run(
|
||
command,
|
||
check=False,
|
||
capture_output=True,
|
||
text=True,
|
||
encoding="utf-8",
|
||
)
|
||
if completed.returncode == 0:
|
||
base_detail = completed.stdout.strip() or "mdi raw completed"
|
||
if request.selected_subpath:
|
||
root_dir = _expected_pdcl_root_dir(output_dir, request.normalized_ref)
|
||
status, detail = _prune_pdcl_download_to_selected_subpath(root_dir, request.selected_subpath, dry_run=False)
|
||
return status, f"{base_detail}\n{detail}", command
|
||
return "downloaded", base_detail, command
|
||
|
||
detail = completed.stderr.strip() or completed.stdout.strip() or "mdi raw failed"
|
||
return "failed", detail, command
|
||
|
||
|
||
def process_issue(
|
||
item: dict,
|
||
output_root: Path,
|
||
dry_run: bool,
|
||
skip_mdi: bool,
|
||
skip_copy: bool,
|
||
only_redownload_affected_cases: bool,
|
||
skip_calib_recovery: bool,
|
||
) -> list[ActionResult]:
|
||
issue_id = int(item["id"])
|
||
issue_name = str(item["name"])
|
||
issue_dir = output_root / normalize_issue_dirname(issue_id)
|
||
|
||
results: list[ActionResult] = []
|
||
seen_pdcl_requests: set[tuple[str, str | None]] = set()
|
||
seen_paths: set[str] = set()
|
||
pdcl_index = 0
|
||
path_index = 0
|
||
|
||
for field_name, raw_value in iter_issue_fields(item):
|
||
if not skip_mdi and not only_redownload_affected_cases:
|
||
for request in extract_pdcl_requests(raw_value):
|
||
request_key = (request.normalized_ref, request.selected_subpath)
|
||
if request_key in seen_pdcl_requests:
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="pdcl_mdi_download",
|
||
raw_value=None if raw_value is None else str(raw_value),
|
||
normalized_ref=request.normalized_ref,
|
||
output_dir=str(issue_dir),
|
||
status="skipped_duplicate",
|
||
detail=(
|
||
f"duplicate PDCL ref: {request.normalized_ref}"
|
||
if request.selected_subpath is None
|
||
else f"duplicate PDCL ref+subpath: {request.normalized_ref} / {request.selected_subpath}"
|
||
),
|
||
selected_subpath=request.selected_subpath,
|
||
)
|
||
)
|
||
continue
|
||
|
||
seen_pdcl_requests.add(request_key)
|
||
pdcl_index += 1
|
||
download_dir = issue_dir / f"pdcl_{pdcl_index:02d}"
|
||
request_desc = request.normalized_ref
|
||
if request.selected_subpath:
|
||
request_desc += f"/{request.selected_subpath}"
|
||
log_progress(
|
||
f"issue_{issue_id} [download] pdcl_{pdcl_index:02d}: {compact_text(request_desc)}"
|
||
)
|
||
status, detail, command = run_mdi_download(request, download_dir, dry_run=dry_run)
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="pdcl_mdi_download",
|
||
raw_value=None if raw_value is None else str(raw_value),
|
||
normalized_ref=request.normalized_ref,
|
||
output_dir=str(download_dir),
|
||
status=status,
|
||
detail=detail,
|
||
command=command,
|
||
selected_subpath=request.selected_subpath,
|
||
)
|
||
)
|
||
|
||
if skip_copy:
|
||
continue
|
||
|
||
for raw_path in extract_standard_paths(raw_value):
|
||
if raw_path in seen_paths:
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="standard_path",
|
||
raw_value=raw_path,
|
||
normalized_ref=None,
|
||
output_dir=str(issue_dir),
|
||
status="skipped_duplicate",
|
||
detail=f"duplicate standard path: {raw_path}",
|
||
)
|
||
)
|
||
continue
|
||
|
||
seen_paths.add(raw_path)
|
||
path_index += 1
|
||
copy_dir = issue_dir / f"path_{path_index:02d}"
|
||
affected_standard_path = is_affected_standard_path(raw_path)
|
||
if only_redownload_affected_cases and not affected_standard_path:
|
||
continue
|
||
|
||
log_progress(
|
||
f"issue_{issue_id} [download] path_{path_index:02d}: {compact_text(raw_path)}"
|
||
)
|
||
resolved_source_path, candidates = resolve_existing_path(raw_path)
|
||
if resolved_source_path is None:
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="standard_path",
|
||
raw_value=raw_path,
|
||
normalized_ref=None,
|
||
output_dir=str(copy_dir),
|
||
status="skipped_missing",
|
||
detail="source path not found after rewrite attempts",
|
||
candidate_paths=[str(candidate) for candidate in candidates],
|
||
)
|
||
)
|
||
continue
|
||
|
||
legacy_target_names = []
|
||
if affected_standard_path:
|
||
legacy_target_names.append(Path(raw_path.strip()).name)
|
||
|
||
status, detail = copy_source_path(
|
||
resolved_source_path,
|
||
copy_dir,
|
||
dry_run=dry_run,
|
||
replace_existing=only_redownload_affected_cases and affected_standard_path,
|
||
legacy_target_names=legacy_target_names,
|
||
)
|
||
target_root = build_copied_target_root(copy_dir, resolved_source_path)
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="standard_path",
|
||
raw_value=raw_path,
|
||
normalized_ref=None,
|
||
output_dir=str(copy_dir),
|
||
status=status,
|
||
detail=detail,
|
||
resolved_source_path=str(resolved_source_path),
|
||
candidate_paths=[str(candidate) for candidate in candidates],
|
||
)
|
||
)
|
||
|
||
sync_status, sync_detail, shared_source_dir = sync_shared_test_data(
|
||
resolved_source_path,
|
||
target_root,
|
||
dry_run=dry_run,
|
||
)
|
||
if sync_status is not None:
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="shared_test_data",
|
||
raw_value=raw_path,
|
||
normalized_ref=None,
|
||
output_dir=str(target_root / "test_data"),
|
||
status=sync_status,
|
||
detail=sync_detail or "",
|
||
resolved_source_path=shared_source_dir,
|
||
candidate_paths=[str(candidate) for candidate in candidates],
|
||
)
|
||
)
|
||
|
||
if not skip_calib_recovery:
|
||
calib_status, calib_detail, calib_source_path = recover_target_root_calib(
|
||
source_root=resolved_source_path,
|
||
target_root=target_root,
|
||
dry_run=dry_run,
|
||
)
|
||
results.append(
|
||
ActionResult(
|
||
issue_id=issue_id,
|
||
issue_name=issue_name,
|
||
source_field=field_name,
|
||
source_kind="case_calib_recovery",
|
||
raw_value=raw_path,
|
||
normalized_ref=None,
|
||
output_dir=str(target_root / "test_data" / "calibs"),
|
||
status=calib_status,
|
||
detail=calib_detail,
|
||
resolved_source_path=calib_source_path,
|
||
candidate_paths=[str(candidate) for candidate in candidates],
|
||
)
|
||
)
|
||
|
||
return results
|
||
|
||
|
||
def build_manifest(
|
||
args: argparse.Namespace,
|
||
input_json: Path,
|
||
output_root: Path,
|
||
action_results: list[ActionResult],
|
||
) -> dict:
|
||
summary: dict[str, int] = {}
|
||
for action in action_results:
|
||
summary[action.status] = summary.get(action.status, 0) + 1
|
||
|
||
return {
|
||
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||
"input_json": str(input_json),
|
||
"output_root": str(output_root),
|
||
"dry_run": args.dry_run,
|
||
"skip_mdi": args.skip_mdi,
|
||
"skip_copy": args.skip_copy,
|
||
"skip_calib_recovery": args.skip_calib_recovery,
|
||
"only_redownload_affected_cases": args.only_redownload_affected_cases,
|
||
"issue_filter": args.issue_ids or [],
|
||
"summary": summary,
|
||
"actions": [action.to_dict() for action in action_results],
|
||
}
|
||
|
||
|
||
def print_summary(manifest: dict) -> None:
|
||
print(f"input_json: {manifest['input_json']}")
|
||
print(f"output_root: {manifest['output_root']}")
|
||
print(f"dry_run: {manifest['dry_run']}")
|
||
for status, count in sorted(manifest["summary"].items()):
|
||
print(f"{status}: {count}")
|
||
|
||
|
||
def main() -> int:
|
||
args = parse_args()
|
||
input_json = Path(args.input_json).resolve()
|
||
output_root = Path(args.output_root).resolve()
|
||
manifest_path = (
|
||
Path(args.manifest_path).resolve()
|
||
if args.manifest_path
|
||
else output_root / "download_manifest.json"
|
||
)
|
||
|
||
items = load_issue_items(input_json)
|
||
if args.issue_ids:
|
||
issue_filter = set(args.issue_ids)
|
||
items = [item for item in items if int(item["id"]) in issue_filter]
|
||
|
||
ensure_dir(output_root, dry_run=args.dry_run)
|
||
|
||
action_results: list[ActionResult] = []
|
||
total_items = len(items)
|
||
log_progress(f"issues_to_process: {total_items}")
|
||
for index, item in enumerate(items, start=1):
|
||
issue_id = int(item["id"])
|
||
issue_name = compact_text(item.get("name"), max_len=64)
|
||
log_progress(f"[{index}/{total_items}] issue_{issue_id} start: {issue_name}")
|
||
issue_results = process_issue(
|
||
item=item,
|
||
output_root=output_root,
|
||
dry_run=args.dry_run,
|
||
skip_mdi=args.skip_mdi,
|
||
skip_copy=args.skip_copy,
|
||
only_redownload_affected_cases=args.only_redownload_affected_cases,
|
||
skip_calib_recovery=args.skip_calib_recovery,
|
||
)
|
||
action_results.extend(issue_results)
|
||
log_progress(
|
||
f"[{index}/{total_items}] issue_{issue_id} done: {summarize_issue_results(issue_results)}"
|
||
)
|
||
|
||
manifest = build_manifest(args, input_json, output_root, action_results)
|
||
ensure_dir(manifest_path.parent, dry_run=args.dry_run)
|
||
if not args.dry_run:
|
||
manifest_path.write_text(
|
||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||
encoding="utf-8",
|
||
)
|
||
|
||
print_summary(manifest)
|
||
if args.dry_run:
|
||
print(f"manifest (not written in dry-run): {manifest_path}")
|
||
else:
|
||
print(f"manifest: {manifest_path}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|