单目3D初始代码

This commit is contained in:
zhao.zhu
2026-06-24 09:35:46 +08:00
commit 04a5895b6b
1153 changed files with 340700 additions and 0 deletions

23
tests/__init__.py Executable file
View File

@@ -0,0 +1,23 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.utils import ASSETS, WEIGHTS_DIR, checks
# Constants used in tests
MODEL = WEIGHTS_DIR / "path with spaces" / "yolo26n.pt" # test spaces in path
CFG = "yolo26n.yaml"
SOURCE = ASSETS / "bus.jpg"
SOURCES_LIST = [ASSETS / "bus.jpg", ASSETS, ASSETS / "*", ASSETS / "**/*.jpg"]
CUDA_IS_AVAILABLE = checks.cuda_is_available()
CUDA_DEVICE_COUNT = checks.cuda_device_count()
TASK_MODEL_DATA = [(task, WEIGHTS_DIR / TASK2MODEL[task], TASK2DATA[task]) for task in TASKS]
MODELS = frozenset([*list(TASK2MODEL.values()), "yolo11n-grayscale.pt"])
__all__ = (
"CFG",
"CUDA_DEVICE_COUNT",
"CUDA_IS_AVAILABLE",
"MODEL",
"SOURCE",
"SOURCES_LIST",
)

59
tests/conftest.py Executable file
View File

@@ -0,0 +1,59 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import shutil
from pathlib import Path
def pytest_addoption(parser):
"""Add custom command-line options to pytest."""
parser.addoption("--slow", action="store_true", default=False, help="Run slow tests")
def pytest_collection_modifyitems(config, items):
"""Modify the list of test items to exclude tests marked as slow if the --slow option is not specified.
Args:
config: The pytest configuration object that provides access to command-line options.
items (list): The list of collected pytest item objects to be modified based on the presence of --slow option.
"""
if not config.getoption("--slow"):
# Remove the item entirely from the list of test items if it's marked as 'slow'
items[:] = [item for item in items if "slow" not in item.keywords]
def pytest_sessionstart(session):
"""Initialize session configurations for pytest.
This function is automatically called by pytest after the 'Session' object has been created but before performing
test collection. It sets the initial seeds for the test session.
Args:
session: The pytest session object.
"""
from ultralytics.utils.torch_utils import init_seeds
init_seeds()
def pytest_terminal_summary(terminalreporter, exitstatus, config):
"""Cleanup operations after pytest session.
This function is automatically called by pytest at the end of the entire test session. It removes certain files and
directories used during testing.
Args:
terminalreporter: The terminal reporter object used for terminal output.
exitstatus (int): The exit status of the test run.
config: The pytest config object.
"""
from ultralytics.utils import WEIGHTS_DIR
# Remove files
models = [path for x in {"*.onnx", "*.torchscript"} for path in WEIGHTS_DIR.rglob(x)]
for file in ["decelera_portrait_min.mov", "bus.jpg", "yolo26n.onnx", "yolo26n.torchscript", *models]:
Path(file).unlink(missing_ok=True)
# Remove directories
models = [path for x in {"*.mlpackage", "*_openvino_model"} for path in WEIGHTS_DIR.rglob(x)]
for directory in [WEIGHTS_DIR / "path with spaces", *models]:
shutil.rmtree(directory, ignore_errors=True)

View File

@@ -0,0 +1,65 @@
import numpy as np
from tools.analyze_mono3d_head_targets import (
activation_lateral_half_span_m,
best_in_box_offset_cells,
expand_bbox_for_assigner,
infer_cut_label,
recommend_l1_norm,
)
def test_expand_bbox_for_assigner_enforces_min_side():
expanded = expand_bbox_for_assigner(np.array([10.0, 10.0, 14.0, 14.0]), 16.0)
assert np.allclose(expanded, np.array([4.0, 4.0, 20.0, 20.0]))
def test_best_in_box_offset_cells_returns_best_case_grid_offset():
offset = best_in_box_offset_cells(
target_uv_px=np.array([40.0, 24.0]),
bbox_xyxy=np.array([16.0, 8.0, 64.0, 40.0]),
img_w=96,
img_h=64,
stride=8,
)
assert np.allclose(offset, np.array([0.5, 0.5]))
def test_infer_cut_label_matches_loss_mapping_for_cut_in_and_cut_out():
cut_in = np.zeros(42, dtype=np.float64)
cut_in[18:24] = -1
cut_in[25] = 0
cut_in[26:32] = -1
cut_in[33] = 0
cut_in[34:40] = -1
cut_in[41] = 0
assert infer_cut_label(cut_in) == 1
cut_out = np.zeros(42, dtype=np.float64)
cut_out[10:16] = -1
cut_out[17] = 0
cut_out[26:32] = -1
cut_out[33] = 0
cut_out[34:40] = -1
cut_out[41] = 0
assert infer_cut_label(cut_out) == 2
def test_recommend_l1_norm_prefers_median_and_robust_scale():
rec = recommend_l1_norm([1.0, 2.0, 3.0, 10.0])
assert rec["offset_median"] == 2.5
assert np.isclose(rec["offset_mean"], 4.0)
assert np.isclose(rec["scale_std"], np.std([1.0, 2.0, 3.0, 10.0], ddof=0))
assert rec["scale_p84_p16_half"] < rec["scale_std"]
def test_activation_lateral_half_span_m_uses_depth_and_fx():
half_span = activation_lateral_half_span_m(
anchor_uv_px=np.array([100.0, 50.0]),
target_v_px=50.0,
stride=8,
calib={"fx": 200.0, "fy": 200.0, "cx": 100.0, "cy": 50.0, "distort_coeffs": []},
depth_metric=25.0,
)
expected = (8.0 * 8.0 / 200.0) * 25.0
assert np.isclose(half_span, expected)

138
tests/test_cli.py Executable file
View File

@@ -0,0 +1,138 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import subprocess
from pathlib import Path
import pytest
from PIL import Image
from tests import CUDA_DEVICE_COUNT, CUDA_IS_AVAILABLE, MODELS, TASK_MODEL_DATA
from ultralytics.utils import ARM64, ASSETS, LINUX, WEIGHTS_DIR, checks
from ultralytics.utils.torch_utils import TORCH_1_11
def run(cmd: str) -> None:
"""Execute a shell command using subprocess."""
subprocess.run(cmd.split(), check=True)
def test_special_modes() -> None:
"""Test various special command-line modes for YOLO functionality."""
run("yolo help")
run("yolo checks")
run("yolo version")
run("yolo settings reset")
run("yolo cfg")
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
def test_train(task: str, model: str, data: str) -> None:
"""Test YOLO training for different tasks, models, and datasets."""
run(f"yolo train {task} model={model} data={data} imgsz=32 epochs=1 cache=disk")
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
def test_val(task: str, model: str, data: str) -> None:
"""Test YOLO validation process for specified task, model, and data using a shell command."""
for end2end in {False, True}:
run(
f"yolo val {task} model={model} data={data} imgsz=32 save_txt save_json visualize end2end={end2end} max_det=100 agnostic_nms"
)
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
def test_predict(task: str, model: str, data: str) -> None:
"""Test YOLO prediction on provided sample assets for specified task and model."""
for end2end in {False, True}:
run(
f"yolo {task} predict model={model} source={ASSETS} imgsz=32 save save_crop save_txt visualize end2end={end2end} max_det=100"
)
@pytest.mark.parametrize("model", MODELS)
def test_export(model: str) -> None:
"""Test exporting a YOLO model to TorchScript format."""
for end2end in {False, True}:
run(f"yolo export model={model} format=torchscript imgsz=32 end2end={end2end} max_det=100")
@pytest.mark.skipif(not TORCH_1_11, reason="RTDETR requires torch>=1.11")
def test_rtdetr(task: str = "detect", model: Path = WEIGHTS_DIR / "rtdetr-l.pt", data: str = "coco8.yaml") -> None:
"""Test the RTDETR functionality within Ultralytics for detection tasks using specified model and data."""
# Add comma, spaces, fraction=0.25 args to test single-image training
run(f"yolo predict {task} model={model} source={ASSETS / 'bus.jpg'} imgsz=160 save save_crop save_txt")
run(f"yolo train {task} model={model} data={data} --imgsz= 160 epochs =1, cache = disk fraction=0.25")
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="MobileSAM with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(
checks.IS_PYTHON_3_8 and LINUX and ARM64,
reason="MobileSAM with CLIP is not supported in Python 3.8 and aarch64 Linux",
)
def test_fastsam(
task: str = "segment", model: str = WEIGHTS_DIR / "FastSAM-s.pt", data: str = "coco8-seg.yaml"
) -> None:
"""Test FastSAM model for segmenting objects in images using various prompts within Ultralytics."""
source = ASSETS / "bus.jpg"
run(f"yolo segment val {task} model={model} data={data} imgsz=32")
run(f"yolo segment predict model={model} source={source} imgsz=32 save save_crop save_txt")
from ultralytics import FastSAM
from ultralytics.models.sam import Predictor
# Create a FastSAM model
sam_model = FastSAM(model) # or FastSAM-x.pt
# Run inference on an image
for s in (source, Image.open(source)):
everything_results = sam_model(s, device="cpu", retina_masks=True, imgsz=320, conf=0.4, iou=0.9)
# Remove small regions
_new_masks, _ = Predictor.remove_small_regions(everything_results[0].masks.data, min_area=20)
# Run inference with bboxes and points and texts prompt at the same time
sam_model(source, bboxes=[439, 437, 524, 709], points=[[200, 200]], labels=[1], texts="a photo of a dog")
def test_mobilesam() -> None:
"""Test MobileSAM segmentation with point and box prompts using Ultralytics."""
from ultralytics import SAM
# Load the model
model = SAM(WEIGHTS_DIR / "mobile_sam.pt")
# Source
source = ASSETS / "zidane.jpg"
# Predict a segment based on a 1D point prompt and 1D labels.
model.predict(source, points=[900, 370], labels=[1])
# Predict a segment based on 3D points and 2D labels (multiple points per object).
model.predict(source, points=[[[900, 370], [1000, 100]]], labels=[[1, 1]])
# Predict a segment based on a box prompt
model.predict(source, bboxes=[439, 437, 524, 709], save=True)
# Predict all
# model(source)
# Slow Tests -----------------------------------------------------------------------------------------------------------
@pytest.mark.slow
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
@pytest.mark.skipif(not CUDA_IS_AVAILABLE, reason="CUDA is not available")
@pytest.mark.skipif(CUDA_DEVICE_COUNT < 2, reason="DDP is not available")
def test_train_gpu(task: str, model: str, data: str) -> None:
"""Test YOLO training on GPU(s) for various tasks and models."""
run(f"yolo train {task} model={model} data={data} imgsz=32 epochs=1 device=0") # single GPU
run(f"yolo train {task} model={model} data={data} imgsz=32 epochs=1 device=0,1") # multi GPU
@pytest.mark.parametrize(
"solution",
["count", "blur", "workout", "heatmap", "isegment", "visioneye", "speed", "queue", "analytics", "trackzone"],
)
def test_solutions(solution: str) -> None:
"""Test yolo solutions command-line modes."""
run(f"yolo solutions {solution} verbose=False")

224
tests/test_cuda.py Executable file
View File

@@ -0,0 +1,224 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import os
from itertools import product
from pathlib import Path
import pytest
import torch
from tests import CUDA_DEVICE_COUNT, CUDA_IS_AVAILABLE, MODEL, SOURCE
from ultralytics import YOLO
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.utils import ASSETS, IS_JETSON, WEIGHTS_DIR
from ultralytics.utils.autodevice import GPUInfo
from ultralytics.utils.checks import check_amp, check_tensorrt
from ultralytics.utils.torch_utils import TORCH_1_13
# Try to find idle devices if CUDA is available
DEVICES = []
if CUDA_IS_AVAILABLE:
if IS_JETSON:
DEVICES = [0] # NVIDIA Jetson only has one GPU and does not fully support pynvml library
else:
gpu_info = GPUInfo()
gpu_info.print_status()
autodevice_fraction = __import__("os").environ.get("YOLO_AUTODEVICE_FRACTION_FREE", 0.3)
if idle_gpus := gpu_info.select_idle_gpu(
count=2,
min_memory_fraction=autodevice_fraction,
min_util_fraction=autodevice_fraction,
):
DEVICES = idle_gpus
def test_checks():
"""Validate CUDA settings against torch CUDA functions."""
assert torch.cuda.is_available() == CUDA_IS_AVAILABLE
assert torch.cuda.device_count() == CUDA_DEVICE_COUNT
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_amp():
"""Test AMP training checks."""
model = YOLO("yolo26n.pt").model.to(f"cuda:{DEVICES[0]}")
assert check_amp(model)
@pytest.mark.slow
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch, simplify, nms",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, simplify, nms)
for task, dynamic, int8, half, batch, simplify, nms in product(
TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False]
)
if not (
(int8 and half) or (task == "classify" and nms) or (task == "obb" and nms and (not TORCH_1_13 or IS_JETSON))
)
],
)
def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms):
"""Test YOLO exports to ONNX format with various configurations and parameters."""
file = YOLO(TASK2MODEL[task]).export(
format="onnx",
imgsz=32,
dynamic=dynamic,
int8=int8,
half=half,
batch=batch,
simplify=simplify,
nms=nms,
device=DEVICES[0],
# opset=20 if nms else None, # fix ONNX Runtime errors with NMS
)
YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32, device=DEVICES[0]) # exported model inference
Path(file).unlink() # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch",
[ # generate all combinations but exclude those where both int8 and half are True
(task, dynamic, int8, half, batch)
# Note: tests reduced below pending compute availability expansion as GPU CI runner utilization is high
# for task, dynamic, int8, half, batch in product(TASKS, [True, False], [True, False], [True, False], [1, 2])
for task, dynamic, int8, half, batch in product(TASKS, [True], [True], [False], [2])
if not (int8 and half) # exclude cases where both int8 and half are True
],
)
def test_export_engine_matrix(task, dynamic, int8, half, batch):
"""Test YOLO model export to TensorRT format for various configurations and run inference."""
check_tensorrt()
import tensorrt as trt
is_trt10 = int(trt.__version__.split(".", 1)[0]) >= 10
if is_trt10 and int8 and dynamic:
pytest.skip("YOLO26 INT8+dynamic export requires explicit quantization on TensorRT 10+")
file = YOLO(TASK2MODEL[task]).export(
format="engine",
imgsz=32,
dynamic=dynamic,
int8=int8,
half=half,
batch=batch,
data=TASK2DATA[task],
workspace=1, # reduce workspace GB for less resource utilization during testing
simplify=True,
device=DEVICES[0],
)
YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32, device=DEVICES[0]) # exported model inference
Path(file).unlink() # cleanup
Path(file).with_suffix(".cache").unlink(missing_ok=True) if int8 else None # cleanup INT8 cache
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_train():
"""Test model training on a minimal dataset using available CUDA devices."""
device = tuple(DEVICES) if len(DEVICES) > 1 else DEVICES[0]
# NVIDIA Jetson only has one GPU and therefore skipping checks
if not IS_JETSON:
results = YOLO(MODEL).train(data="coco8-grayscale.yaml", imgsz=64, epochs=1, device=DEVICES[0], batch=-1)
results = YOLO(MODEL).train(data="coco8.yaml", imgsz=64, epochs=1, device=device, batch=15, compile=True)
results = YOLO(MODEL).train(data="coco128.yaml", imgsz=64, epochs=1, device=device, batch=15, val=False)
visible = eval(os.environ["CUDA_VISIBLE_DEVICES"])
assert visible == device, f"Passed GPUs '{device}', but used GPUs '{visible}'"
# Note DDP training returns None, single-GPU returns metrics
assert (results is None) if len(DEVICES) > 1 else (results is not None)
@pytest.mark.slow
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_predict_multiple_devices():
"""Validate model prediction consistency across CPU and CUDA devices."""
model = YOLO("yolo26n.pt")
# Test CPU
model = model.cpu()
assert str(model.device) == "cpu"
_ = model(SOURCE)
assert str(model.device) == "cpu"
# Test CUDA
cuda_device = f"cuda:{DEVICES[0]}"
model = model.to(cuda_device)
assert str(model.device) == cuda_device
_ = model(SOURCE)
assert str(model.device) == cuda_device
# Test CPU again
model = model.cpu()
assert str(model.device) == "cpu"
_ = model(SOURCE)
assert str(model.device) == "cpu"
# Test CUDA again
model = model.to(cuda_device)
assert str(model.device) == cuda_device
_ = model(SOURCE)
assert str(model.device) == cuda_device
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_autobatch():
"""Check optimal batch size for YOLO model training using autobatch utility."""
from ultralytics.utils.autobatch import check_train_batch_size
check_train_batch_size(YOLO(MODEL).model.to(f"cuda:{DEVICES[0]}"), imgsz=128, amp=True)
@pytest.mark.slow
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_utils_benchmarks():
"""Profile YOLO models for performance benchmarks."""
from ultralytics.utils.benchmarks import ProfileModels
# Pre-export a dynamic engine model to use dynamic inference
YOLO(MODEL).export(format="engine", imgsz=32, dynamic=True, batch=1, device=DEVICES[0])
ProfileModels(
[MODEL],
imgsz=32,
half=False,
min_time=1,
num_timed_runs=3,
num_warmup_runs=1,
device=DEVICES[0],
).run()
@pytest.mark.skipif(not DEVICES, reason="No CUDA devices available")
def test_predict_sam():
"""Test SAM model predictions using different prompts."""
from ultralytics import SAM
from ultralytics.models.sam import Predictor as SAMPredictor
model = SAM(WEIGHTS_DIR / "sam2.1_b.pt")
model.info()
# Run inference with various prompts
model(SOURCE, device=DEVICES[0])
model(SOURCE, bboxes=[439, 437, 524, 709], device=DEVICES[0])
model(ASSETS / "zidane.jpg", points=[900, 370], device=DEVICES[0])
model(ASSETS / "zidane.jpg", points=[900, 370], labels=[1], device=DEVICES[0])
model(ASSETS / "zidane.jpg", points=[[900, 370]], labels=[1], device=DEVICES[0])
model(ASSETS / "zidane.jpg", points=[[400, 370], [900, 370]], labels=[1, 1], device=DEVICES[0])
model(ASSETS / "zidane.jpg", points=[[[900, 370], [1000, 100]]], labels=[[1, 1]], device=DEVICES[0])
# Test predictor
predictor = SAMPredictor(
overrides=dict(
conf=0.25,
task="segment",
mode="predict",
imgsz=1024,
model=WEIGHTS_DIR / "mobile_sam.pt",
device=DEVICES[0],
)
)
predictor.set_image(ASSETS / "zidane.jpg")
# predictor(bboxes=[439, 437, 524, 709])
# predictor(points=[900, 370], labels=[1])
predictor.reset_image()

296
tests/test_engine.py Executable file
View File

@@ -0,0 +1,296 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import sys
from unittest import mock
import numpy as np
import torch
from tests import MODEL, SOURCE
from ultralytics import YOLO
from ultralytics.cfg import get_cfg
from ultralytics.engine.exporter import Exporter
from ultralytics.models.yolo import classify, detect, segment
from ultralytics.utils import ASSETS, DEFAULT_CFG, WEIGHTS_DIR
def test_func(*args, **kwargs):
"""Test function used as a callback stub to verify callback registration."""
print("callback test passed")
def test_export():
"""Test model exporting functionality by adding a callback and verifying its execution."""
exporter = Exporter()
exporter.add_callback("on_export_start", test_func)
assert test_func in exporter.callbacks["on_export_start"], "callback test failed"
f = exporter(model=YOLO("yolo26n.yaml").model)
YOLO(f)(SOURCE) # exported model inference
def test_detect():
"""Test YOLO object detection training, validation, and prediction functionality."""
overrides = {"data": "coco8.yaml", "model": "yolo26n.yaml", "imgsz": 32, "epochs": 1, "save": False}
cfg = get_cfg(DEFAULT_CFG)
cfg.data = "coco8.yaml"
cfg.imgsz = 32
# Trainer
trainer = detect.DetectionTrainer(overrides=overrides)
trainer.add_callback("on_train_start", test_func)
assert test_func in trainer.callbacks["on_train_start"], "callback test failed"
trainer.train()
# Validator
val = detect.DetectionValidator(args=cfg)
val.add_callback("on_val_start", test_func)
assert test_func in val.callbacks["on_val_start"], "callback test failed"
val(model=trainer.best) # validate best.pt
# Predictor
pred = detect.DetectionPredictor(overrides={"imgsz": [64, 64]})
pred.add_callback("on_predict_start", test_func)
assert test_func in pred.callbacks["on_predict_start"], "callback test failed"
# Confirm there is no issue with sys.argv being empty
with mock.patch.object(sys, "argv", []):
result = pred(source=ASSETS, model=MODEL)
assert len(result), "predictor test failed"
# Test resume functionality
overrides["resume"] = trainer.last
trainer = detect.DetectionTrainer(overrides=overrides)
try:
trainer.train()
except Exception as e:
print(f"Expected exception caught: {e}")
return
raise Exception("Resume test failed!")
def test_segment():
"""Test image segmentation training, validation, and prediction pipelines using YOLO models."""
overrides = {
"data": "coco8-seg.yaml",
"model": "yolo26n-seg.yaml",
"imgsz": 32,
"epochs": 1,
"save": False,
"mask_ratio": 1,
"overlap_mask": False,
}
cfg = get_cfg(DEFAULT_CFG)
cfg.data = "coco8-seg.yaml"
cfg.imgsz = 32
# Trainer
trainer = segment.SegmentationTrainer(overrides=overrides)
trainer.add_callback("on_train_start", test_func)
assert test_func in trainer.callbacks["on_train_start"], "callback test failed"
trainer.train()
# Validator
val = segment.SegmentationValidator(args=cfg)
val.add_callback("on_val_start", test_func)
assert test_func in val.callbacks["on_val_start"], "callback test failed"
val(model=trainer.best) # validate best.pt
# Predictor
pred = segment.SegmentationPredictor(overrides={"imgsz": [64, 64]})
pred.add_callback("on_predict_start", test_func)
assert test_func in pred.callbacks["on_predict_start"], "callback test failed"
result = pred(source=ASSETS, model=WEIGHTS_DIR / "yolo26n-seg.pt")
assert len(result), "predictor test failed"
# Test resume functionality
overrides["resume"] = trainer.last
trainer = segment.SegmentationTrainer(overrides=overrides)
try:
trainer.train()
except Exception as e:
print(f"Expected exception caught: {e}")
return
raise Exception("Resume test failed!")
def test_classify():
"""Test image classification including training, validation, and prediction phases."""
overrides = {"data": "imagenet10", "model": "yolo26n-cls.yaml", "imgsz": 32, "epochs": 1, "save": False}
cfg = get_cfg(DEFAULT_CFG)
cfg.data = "imagenet10"
cfg.imgsz = 32
# Trainer
trainer = classify.ClassificationTrainer(overrides=overrides)
trainer.add_callback("on_train_start", test_func)
assert test_func in trainer.callbacks["on_train_start"], "callback test failed"
trainer.train()
# Validator
val = classify.ClassificationValidator(args=cfg)
val.add_callback("on_val_start", test_func)
assert test_func in val.callbacks["on_val_start"], "callback test failed"
val(model=trainer.best)
# Predictor
pred = classify.ClassificationPredictor(overrides={"imgsz": [64, 64]})
pred.add_callback("on_predict_start", test_func)
assert test_func in pred.callbacks["on_predict_start"], "callback test failed"
result = pred(source=ASSETS, model=trainer.best)
assert len(result), "predictor test failed"
def test_nan_recovery():
"""Test NaN loss detection and recovery during training."""
nan_injected = [False]
def inject_nan(trainer):
"""Inject NaN into loss during batch processing to test recovery mechanism."""
if trainer.epoch == 1 and trainer.tloss is not None and not nan_injected[0]:
trainer.tloss *= torch.tensor(float("nan"))
nan_injected[0] = True
overrides = {"data": "coco8.yaml", "model": "yolo26n.yaml", "imgsz": 32, "epochs": 3}
trainer = detect.DetectionTrainer(overrides=overrides)
trainer.add_callback("on_train_batch_end", inject_nan)
trainer.train()
assert nan_injected[0], "NaN injection failed"
def test_ground3d_validator_prints_combined_metrics(caplog):
"""Test Ground3DDetectionValidator prints whole and face 3D metrics."""
from ultralytics.models.yolo.detect.train import Ground3DDetectionValidator
validator = Ground3DDetectionValidator(args=get_cfg(DEFAULT_CFG))
validator.names = {0: "car"}
validator.args.task = "detect"
validator.args.verbose = False
validator.training = True
validator.seen = 3531
validator.nc = 1
validator.stats = []
validator.metrics.nt_per_class = torch.tensor([59691])
validator.metrics.mean_results = lambda: [0.73, 0.585, 0.657, 0.499]
validator.metrics.keys = ["metrics/precision(B)", "metrics/recall(B)", "metrics/mAP50(B)", "metrics/mAP50-95(B)"]
validator.metrics_3d_results = {
"whole": {
"depth_abs": 48.72,
"depth_rel": 1.915,
"depth_rmse": 54.22,
"center": 52.01,
"uv": 17.35,
"orient": 85.47,
"size": 2.011,
"matched": 5738,
},
"face": {
"depth_abs": 12.31,
"depth_rel": 0.441,
"depth_rmse": 18.22,
"center": 14.07,
"uv": 9.56,
"matched": 2814,
},
}
with caplog.at_level("INFO", logger="ultralytics"):
validator.print_results()
all_lines = [record.message for record in caplog.records if "all" in record.message]
assert len(all_lines) == 1
all_line = all_lines[0]
assert "0.73" in all_line
assert "48.7" in all_line
assert "1.92" in all_line
assert "54.2" in all_line
assert "52" in all_line
assert "17.4" in all_line
assert "85.5" in all_line
assert "2.01" in all_line
assert "5738" in all_line
face_lines = [record.message for record in caplog.records if "3d-face" in record.message]
assert len(face_lines) == 1
face_line = face_lines[0]
assert "12.3" in face_line
assert "0.441" in face_line
assert "18.2" in face_line
assert "14.1" in face_line
assert "9.56" in face_line
assert "2814" in face_line
def test_ground3d_validator_prints_invalid_visible_yaw_as_dash(caplog):
"""Test Ground3DDetectionValidator renders invalid visible-yaw metrics as '-'."""
from ultralytics.models.yolo.detect.train import Ground3DDetectionValidator
validator = Ground3DDetectionValidator(args=get_cfg(DEFAULT_CFG))
validator.names = {0: "car"}
validator.args.task = "detect"
validator.args.verbose = False
validator.training = True
validator.seen = 1
validator.nc = 1
validator.stats = []
validator.metrics.nt_per_class = torch.tensor([1])
validator.metrics.mean_results = lambda: [0.73, 0.585, 0.657, 0.499]
validator.metrics.keys = ["metrics/precision(B)", "metrics/recall(B)", "metrics/mAP50(B)", "metrics/mAP50-95(B)"]
validator.metrics_3d_results = {
"whole": {
"depth_abs": 1.0,
"depth_rel": 0.1,
"depth_rmse": 1.5,
"center": 2.0,
"uv": 3.0,
"orient": 4.0,
"size": 0.5,
"matched": 1,
},
"face": {
"depth_abs": 2.0,
"depth_rel": 0.2,
"depth_rmse": 2.5,
"center": 4.0,
"uv": 5.0,
"size": 0.6,
"direct_orient_visible": np.nan,
"edge_orient_visible": np.nan,
"matched": 1,
},
}
with caplog.at_level("INFO", logger="ultralytics"):
validator.print_results()
all_lines = [record.message for record in caplog.records if "all-3d" in record.message]
assert len(all_lines) == 1
assert " nan" not in all_lines[0]
assert all_lines[0].rstrip().endswith("-")
def test_detect3d_select_topk_3d_metadata_keeps_batch_alignment():
"""Test Detect3D keeps selected 3D predictions, anchors, and strides aligned per sample."""
from ultralytics.nn.modules.head import Detect3D
detect3d = Detect3D(nc=2, reg_max=1, end2end=True, ch=(16,))
detect3d.anchors = torch.tensor([[0.5, 1.5, 2.5, 3.5], [10.5, 11.5, 12.5, 13.5]], dtype=torch.float32)
detect3d.strides = torch.tensor([[8.0, 16.0, 32.0, 64.0]], dtype=torch.float32)
preds_3d = torch.arange(2 * detect3d.no_3d * 4, dtype=torch.float32).reshape(2, detect3d.no_3d, 4)
idx = torch.tensor([[[0], [2]], [[1], [3]]], dtype=torch.long)
preds_sel, anchors_sel, strides_sel = detect3d._select_topk_3d_metadata(preds_3d, idx)
assert preds_sel.shape == (2, 2, detect3d.no_3d)
assert anchors_sel.shape == (2, 2, 2)
assert strides_sel.shape == (2, 2)
torch.testing.assert_close(preds_sel[0, 0], preds_3d[0, :, 0])
torch.testing.assert_close(preds_sel[0, 1], preds_3d[0, :, 2])
torch.testing.assert_close(preds_sel[1, 0], preds_3d[1, :, 1])
torch.testing.assert_close(preds_sel[1, 1], preds_3d[1, :, 3])
torch.testing.assert_close(anchors_sel[0], detect3d.anchors[:, [0, 2]])
torch.testing.assert_close(anchors_sel[1], detect3d.anchors[:, [1, 3]])
torch.testing.assert_close(strides_sel[0], detect3d.strides[0, [0, 2]])
torch.testing.assert_close(strides_sel[1], detect3d.strides[0, [1, 3]])

343
tests/test_exports.py Executable file
View File

@@ -0,0 +1,343 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import io
import shutil
import uuid
from contextlib import redirect_stderr, redirect_stdout
from itertools import product
from pathlib import Path
import pytest
from tests import MODEL, SOURCE
from ultralytics import YOLO
from ultralytics.cfg import TASK2DATA, TASK2MODEL, TASKS
from ultralytics.utils import ARM64, IS_RASPBERRYPI, LINUX, MACOS, MACOS_VERSION, WINDOWS, checks
from ultralytics.utils.torch_utils import TORCH_1_10, TORCH_1_11, TORCH_1_13, TORCH_2_0, TORCH_2_1, TORCH_2_8, TORCH_2_9
@pytest.mark.parametrize("end2end", [False, True])
def test_export_torchscript(end2end):
"""Test YOLO model export to TorchScript format for compatibility and correctness."""
file = YOLO(MODEL).export(format="torchscript", optimize=False, imgsz=32, end2end=end2end)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.parametrize("end2end", [False, True])
def test_export_onnx(end2end):
"""Test YOLO model export to ONNX format with dynamic axes."""
file = YOLO(MODEL).export(format="onnx", dynamic=True, imgsz=32, end2end=end2end)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.skipif(not TORCH_2_1, reason="OpenVINO requires torch>=2.1")
@pytest.mark.parametrize("end2end", [False, True])
def test_export_openvino(end2end):
"""Test YOLO export to OpenVINO format for model inference compatibility."""
file = YOLO(MODEL).export(format="openvino", imgsz=32, end2end=end2end)
if WINDOWS:
# Ensure a unique export path per test to prevent OpenVINO file writes
file = Path(file)
file = file.rename(file.with_stem(f"{file.stem}-{uuid.uuid4()}"))
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_1, reason="OpenVINO requires torch>=2.1")
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch, nms, end2end",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [True, False], [True, False], [True, False], [1, 2], [True, False], [True]
)
if not ((int8 and half) or (task == "classify" and nms) or (end2end and nms))
],
)
# disable end2end=False test for now due to github runner OOM during openvino tests
def test_export_openvino_matrix(task, dynamic, int8, half, batch, nms, end2end):
"""Test YOLO model export to OpenVINO under various configuration matrix conditions."""
file = YOLO(TASK2MODEL[task]).export(
format="openvino",
imgsz=32,
dynamic=dynamic,
int8=int8,
half=half,
batch=batch,
data=TASK2DATA[task],
nms=nms,
end2end=end2end,
)
if WINDOWS:
# Use unique filenames due to Windows file permissions bug possibly due to latent threaded use
file = Path(file)
file = file.rename(file.with_stem(f"{file.stem}-{uuid.uuid4()}"))
YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32, batch=batch) # exported model inference
shutil.rmtree(file, ignore_errors=True) # retry in case of potential lingering multi-threaded file usage errors
@pytest.mark.slow
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch, simplify, nms, end2end",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, simplify, nms, end2end)
for task, dynamic, int8, half, batch, simplify, nms, end2end in product(
TASKS, [True, False], [False], [False], [1, 2], [True, False], [True, False], [True, False]
)
if not ((int8 and half) or (task == "classify" and nms) or (nms and not TORCH_1_13) or (end2end and nms))
],
)
def test_export_onnx_matrix(task, dynamic, int8, half, batch, simplify, nms, end2end):
"""Test YOLO export to ONNX format with various configurations and parameters."""
file = YOLO(TASK2MODEL[task]).export(
format="onnx",
imgsz=32,
dynamic=dynamic,
int8=int8,
half=half,
batch=batch,
simplify=simplify,
nms=nms,
end2end=end2end,
)
YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32) # exported model inference
Path(file).unlink() # cleanup
@pytest.mark.slow
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch, nms, end2end",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [False, True], [False], [False, True], [1, 2], [True, False], [True, False]
)
if not ((task == "classify" and nms) or (end2end and nms))
],
)
def test_export_torchscript_matrix(task, dynamic, int8, half, batch, nms, end2end):
"""Test YOLO model export to TorchScript format under varied configurations."""
file = YOLO(TASK2MODEL[task]).export(
format="torchscript", imgsz=32, dynamic=dynamic, int8=int8, half=half, batch=batch, nms=nms, end2end=end2end
)
YOLO(file)([SOURCE] * batch, imgsz=64 if dynamic else 32) # exported model inference
Path(file).unlink() # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not MACOS, reason="CoreML inference only supported on macOS")
@pytest.mark.skipif(not TORCH_1_11, reason="CoreML export requires torch>=1.11")
@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13")
@pytest.mark.skipif(
MACOS and MACOS_VERSION and MACOS_VERSION >= "15", reason="CoreML YOLO26 matrix test crashes on macOS 15+"
)
@pytest.mark.parametrize(
"task, dynamic, int8, half, nms, batch, end2end",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, nms, batch, end2end)
for task, dynamic, int8, half, nms, batch, end2end in product(
TASKS, [True, False], [True, False], [True, False], [True, False], [1], [True, False]
)
if not (int8 and half)
and not (task != "detect" and nms)
and not (dynamic and nms)
and not (task == "classify" and dynamic)
and not (end2end and nms)
],
)
def test_export_coreml_matrix(task, dynamic, int8, half, nms, batch, end2end):
"""Test YOLO export to CoreML format with various parameter configurations."""
file = YOLO(TASK2MODEL[task]).export(
format="coreml",
imgsz=32,
dynamic=dynamic,
int8=int8,
half=half,
batch=batch,
nms=nms,
end2end=end2end,
)
YOLO(file)([SOURCE] * batch, imgsz=32) # exported model inference
shutil.rmtree(file) # cleanup
@pytest.mark.slow
@pytest.mark.skipif(
not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_1_13, reason="TFLite export requires Python>=3.10 and torch>=1.13"
)
@pytest.mark.skipif(
not LINUX or IS_RASPBERRYPI,
reason="Test disabled as TF suffers from install conflicts on Windows, macOS and Raspberry Pi",
)
@pytest.mark.parametrize(
"task, dynamic, int8, half, batch, nms, end2end",
[ # generate all combinations except for exclusion cases
(task, dynamic, int8, half, batch, nms, end2end)
for task, dynamic, int8, half, batch, nms, end2end in product(
TASKS, [False], [True, False], [True, False], [1], [True, False], [True, False]
)
if not (
(int8 and half)
or (task == "classify" and nms)
or (ARM64 and nms)
or (nms and not TORCH_1_13)
or (end2end and nms)
)
],
)
def test_export_tflite_matrix(task, dynamic, int8, half, batch, nms, end2end):
"""Test YOLO export to TFLite format considering various export configurations."""
file = YOLO(TASK2MODEL[task]).export(
format="tflite", imgsz=32, dynamic=dynamic, int8=int8, half=half, batch=batch, nms=nms, end2end=end2end
)
YOLO(file)([SOURCE] * batch, imgsz=32) # exported model inference
Path(file).unlink() # cleanup
@pytest.mark.skipif(not TORCH_1_11, reason="CoreML export requires torch>=1.11")
@pytest.mark.skipif(WINDOWS, reason="CoreML not supported on Windows") # RuntimeError: BlobWriter not loaded
@pytest.mark.skipif(LINUX and ARM64, reason="CoreML not supported on aarch64 Linux")
@pytest.mark.skipif(checks.IS_PYTHON_3_13, reason="CoreML not supported in Python 3.13")
def test_export_coreml():
"""Test YOLO export to CoreML format and check for errors."""
# Capture stdout and stderr
stdout, stderr = io.StringIO(), io.StringIO()
with redirect_stdout(stdout), redirect_stderr(stderr):
YOLO(MODEL).export(format="coreml", nms=True, imgsz=32)
if MACOS:
file = YOLO(MODEL).export(format="coreml", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # model prediction only supported on macOS for nms=False models
# Check captured output for errors
output = stdout.getvalue() + stderr.getvalue()
assert "Error" not in output, f"CoreML export produced errors: {output}"
assert "You will not be able to run predict()" not in output, "CoreML export has predict() error"
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10, reason="TFLite export requires Python>=3.10")
@pytest.mark.skipif(not TORCH_1_13, reason="TFLite export requires torch>=1.13")
@pytest.mark.skipif(not LINUX, reason="Test disabled as TF suffers from install conflicts on Windows and macOS")
def test_export_tflite():
"""Test YOLO export to TFLite format under specific OS and Python version conditions."""
model = YOLO(MODEL)
file = model.export(format="tflite", imgsz=32)
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.skipif(True, reason="Test disabled")
@pytest.mark.skipif(not LINUX, reason="TF suffers from install conflicts on Windows and macOS")
def test_export_pb():
"""Test YOLO export to TensorFlow's Protobuf (*.pb) format."""
model = YOLO(MODEL)
file = model.export(format="pb", imgsz=32)
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.skipif(True, reason="Test disabled as Paddle protobuf and ONNX protobuf requirements conflict.")
def test_export_paddle():
"""Test YOLO export to Paddle format, noting protobuf conflicts with ONNX."""
YOLO(MODEL).export(format="paddle", imgsz=32)
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_1_10, reason="MNN export requires torch>=1.10")
def test_export_mnn():
"""Test YOLO export to MNN format (WARNING: MNN test must precede NCNN test or CI error on Windows)."""
file = YOLO(MODEL).export(format="mnn", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_1_10, reason="MNN export requires torch>=1.10")
@pytest.mark.parametrize(
"task, int8, half, batch, end2end",
[ # generate all combinations except for exclusion cases
(task, int8, half, batch, end2end)
for task, int8, half, batch, end2end in product(TASKS, [True, False], [True, False], [1, 2], [True, False])
if not (int8 and half)
],
)
def test_export_mnn_matrix(task, int8, half, batch, end2end):
"""Test YOLO export to MNN format considering various export configurations."""
file = YOLO(TASK2MODEL[task]).export(format="mnn", imgsz=32, int8=int8, half=half, batch=batch, end2end=end2end)
YOLO(file)([SOURCE] * batch, imgsz=32) # exported model inference
Path(file).unlink() # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0")
def test_export_ncnn():
"""Test YOLO export to NCNN format."""
file = YOLO(MODEL).export(format="ncnn", imgsz=32)
YOLO(file)(SOURCE, imgsz=32) # exported model inference
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_0, reason="NCNN inference causes segfault on PyTorch<2.0")
@pytest.mark.parametrize("task, half, batch", list(product(TASKS, [True, False], [1])))
def test_export_ncnn_matrix(task, half, batch):
"""Test YOLO export to NCNN format considering various export configurations."""
file = YOLO(TASK2MODEL[task]).export(format="ncnn", imgsz=32, half=half, batch=batch)
YOLO(file)([SOURCE] * batch, imgsz=32) # exported model inference
shutil.rmtree(file, ignore_errors=True) # retry in case of potential lingering multi-threaded file usage errors
@pytest.mark.skipif(not TORCH_2_9, reason="IMX export requires torch>=2.9.0")
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_9, reason="Requires Python>=3.9")
@pytest.mark.skipif(not LINUX, reason="IMX export only supported on Linux")
@pytest.mark.skipif(
IS_RASPBERRYPI, reason="Test disabled as IMX export suffers from OOM (Out of Memory) on Raspberry Pi 5 16GB"
)
def test_export_imx():
"""Test YOLO export to IMX format."""
model = YOLO("yolo11n.pt") # IMX export only supports YOLO11
file = model.export(format="imx", imgsz=32)
YOLO(file)(SOURCE, imgsz=32)
@pytest.mark.slow
@pytest.mark.skipif(not TORCH_2_8, reason="Axelera export requires torch>=2.8.0")
@pytest.mark.skipif(not LINUX, reason="Axelera export only supported on Linux")
@pytest.mark.skipif(not checks.IS_PYTHON_3_10, reason="Axelera export requires Python 3.10")
def test_export_axelera():
"""Test YOLO export to Axelera format."""
# For faster testing, use a smaller calibration dataset (32 image size crashes axelera export, so 64 is used)
file = YOLO(MODEL).export(format="axelera", imgsz=64, data="coco8.yaml")
assert Path(file).exists(), f"Axelera export failed, directory not found: {file}"
shutil.rmtree(file, ignore_errors=True) # cleanup
# @pytest.mark.skipif(True, reason="Disabled for debugging ruamel.yaml installation required by executorch")
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0")
@pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows")
def test_export_executorch():
"""Test YOLO model export to ExecuTorch format."""
file = YOLO(MODEL).export(format="executorch", imgsz=32)
assert Path(file).exists(), f"ExecuTorch export failed, directory not found: {file}"
# Check that .pte file exists in the exported directory
pte_file = Path(file) / Path(MODEL).with_suffix(".pte").name
assert pte_file.exists(), f"ExecuTorch .pte file not found: {pte_file}"
# Check that metadata.yaml exists
metadata_file = Path(file) / "metadata.yaml"
assert metadata_file.exists(), f"ExecuTorch metadata.yaml not found: {metadata_file}"
# Note: Inference testing skipped as ExecuTorch requires special runtime setup
shutil.rmtree(file, ignore_errors=True) # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not checks.IS_PYTHON_MINIMUM_3_10 or not TORCH_2_9, reason="Requires Python>=3.10 and Torch>=2.9.0")
@pytest.mark.skipif(WINDOWS, reason="Skipping test on Windows")
@pytest.mark.parametrize("task", TASKS)
def test_export_executorch_matrix(task):
"""Test YOLO export to ExecuTorch format for various task types."""
file = YOLO(TASK2MODEL[task]).export(format="executorch", imgsz=32)
assert Path(file).exists(), f"ExecuTorch export failed for task '{task}', directory not found: {file}"
# Check that .pte file exists in the exported directory
model_name = Path(TASK2MODEL[task]).with_suffix(".pte").name
pte_file = Path(file) / model_name
assert pte_file.exists(), f"ExecuTorch .pte file not found for task '{task}': {pte_file}"
# Check that metadata.yaml exists
metadata_file = Path(file) / "metadata.yaml"
assert metadata_file.exists(), f"ExecuTorch metadata.yaml not found for task '{task}': {metadata_file}"
# Note: Inference testing skipped as ExecuTorch requires special runtime setup
shutil.rmtree(file, ignore_errors=True) # cleanup

151
tests/test_integrations.py Executable file
View File

@@ -0,0 +1,151 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import contextlib
import os
import subprocess
import time
from pathlib import Path
import pytest
from tests import MODEL, SOURCE
from ultralytics import YOLO, download
from ultralytics.utils import ASSETS_URL, DATASETS_DIR, SETTINGS
from ultralytics.utils.checks import check_requirements
@pytest.mark.slow
def test_tensorboard():
"""Test training with TensorBoard logging enabled."""
SETTINGS["tensorboard"] = True
YOLO("yolo26n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=3, plots=False, device="cpu")
SETTINGS["tensorboard"] = False
@pytest.mark.skipif(not check_requirements("ray", install=False), reason="ray[tune] not installed")
def test_model_ray_tune():
"""Tune YOLO model using Ray for hyperparameter optimization."""
YOLO("yolo26n-cls.yaml").tune(
use_ray=True, data="imagenet10", grace_period=1, iterations=1, imgsz=32, epochs=1, plots=False, device="cpu"
)
@pytest.mark.skipif(not check_requirements("mlflow", install=False), reason="mlflow not installed")
def test_mlflow():
"""Test training with MLflow tracking enabled."""
SETTINGS["mlflow"] = True
YOLO("yolo26n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=3, plots=False, device="cpu")
SETTINGS["mlflow"] = False
@pytest.mark.skipif(True, reason="Test failing in scheduled CI https://github.com/ultralytics/ultralytics/pull/8868")
@pytest.mark.skipif(not check_requirements("mlflow", install=False), reason="mlflow not installed")
def test_mlflow_keep_run_active():
"""Ensure MLflow run status matches MLFLOW_KEEP_RUN_ACTIVE environment variable settings."""
import mlflow
SETTINGS["mlflow"] = True
run_name = "Test Run"
os.environ["MLFLOW_RUN"] = run_name
# Test with MLFLOW_KEEP_RUN_ACTIVE=True
os.environ["MLFLOW_KEEP_RUN_ACTIVE"] = "True"
YOLO("yolo26n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=1, plots=False, device="cpu")
status = mlflow.active_run().info.status
assert status == "RUNNING", "MLflow run should be active when MLFLOW_KEEP_RUN_ACTIVE=True"
run_id = mlflow.active_run().info.run_id
# Test with MLFLOW_KEEP_RUN_ACTIVE=False
os.environ["MLFLOW_KEEP_RUN_ACTIVE"] = "False"
YOLO("yolo26n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=1, plots=False, device="cpu")
status = mlflow.get_run(run_id=run_id).info.status
assert status == "FINISHED", "MLflow run should be ended when MLFLOW_KEEP_RUN_ACTIVE=False"
# Test with MLFLOW_KEEP_RUN_ACTIVE not set
os.environ.pop("MLFLOW_KEEP_RUN_ACTIVE", None)
YOLO("yolo26n-cls.yaml").train(data="imagenet10", imgsz=32, epochs=1, plots=False, device="cpu")
status = mlflow.get_run(run_id=run_id).info.status
assert status == "FINISHED", "MLflow run should be ended by default when MLFLOW_KEEP_RUN_ACTIVE is not set"
SETTINGS["mlflow"] = False
@pytest.mark.skipif(not check_requirements("tritonclient", install=False), reason="tritonclient[all] not installed")
def test_triton(tmp_path):
"""Test NVIDIA Triton Server functionalities with YOLO model."""
check_requirements("tritonclient[all]")
from tritonclient.http import InferenceServerClient
# Create variables
model_name = "yolo"
triton_repo = tmp_path / "triton_repo" # Triton repo path
triton_model = triton_repo / model_name # Triton model path
# Export model to ONNX
f = YOLO(MODEL).export(format="onnx", dynamic=True)
# Prepare Triton repo
(triton_model / "1").mkdir(parents=True, exist_ok=True)
Path(f).rename(triton_model / "1" / "model.onnx")
(triton_model / "config.pbtxt").touch()
# Define image https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tritonserver
tag = "nvcr.io/nvidia/tritonserver:23.09-py3" # 6.4 GB
# Pull the image
subprocess.call(f"docker pull {tag}", shell=True)
# Run the Triton server and capture the container ID
container_id = (
subprocess.check_output(
f"docker run -d --rm -v {triton_repo}:/models -p 8000:8000 {tag} tritonserver --model-repository=/models",
shell=True,
)
.decode("utf-8")
.strip()
)
# Wait for the Triton server to start
triton_client = InferenceServerClient(url="localhost:8000", verbose=False, ssl=False)
# Wait until model is ready
for _ in range(10):
with contextlib.suppress(Exception):
assert triton_client.is_model_ready(model_name)
break
time.sleep(1)
# Check Triton inference
YOLO(f"http://localhost:8000/{model_name}", "detect")(SOURCE) # exported model inference
# Kill and remove the container at the end of the test
subprocess.call(f"docker kill {container_id}", shell=True)
@pytest.mark.skipif(not check_requirements("faster-coco-eval", install=False), reason="faster-coco-eval not installed")
def test_faster_coco_eval():
"""Validate YOLO model predictions on COCO dataset using faster-coco-eval."""
from ultralytics.models.yolo.detect import DetectionValidator
from ultralytics.models.yolo.pose import PoseValidator
from ultralytics.models.yolo.segment import SegmentationValidator
args = {"model": "yolo26n.pt", "data": "coco8.yaml", "save_json": True, "imgsz": 64}
validator = DetectionValidator(args=args)
validator()
validator.is_coco = True
download(f"{ASSETS_URL}/instances_val2017.json", dir=DATASETS_DIR / "coco8/annotations")
_ = validator.eval_json(validator.stats)
args = {"model": "yolo26n-seg.pt", "data": "coco8-seg.yaml", "save_json": True, "imgsz": 64}
validator = SegmentationValidator(args=args)
validator()
validator.is_coco = True
download(f"{ASSETS_URL}/instances_val2017.json", dir=DATASETS_DIR / "coco8-seg/annotations")
_ = validator.eval_json(validator.stats)
args = {"model": "yolo26n-pose.pt", "data": "coco8-pose.yaml", "save_json": True, "imgsz": 64}
validator = PoseValidator(args=args)
validator()
validator.is_coco = True
download(f"{ASSETS_URL}/person_keypoints_val2017.json", dir=DATASETS_DIR / "coco8-pose/annotations")
_ = validator.eval_json(validator.stats)

1680
tests/test_metrics_3d.py Executable file

File diff suppressed because it is too large Load Diff

809
tests/test_python.py Executable file
View File

@@ -0,0 +1,809 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
import contextlib
import csv
import urllib
from copy import copy
from pathlib import Path
import cv2
import numpy as np
import pytest
import torch
from PIL import Image
from tests import CFG, MODEL, MODELS, SOURCE, SOURCES_LIST, TASK_MODEL_DATA
from ultralytics import RTDETR, YOLO
from ultralytics.cfg import TASK2DATA, TASKS
from ultralytics.data.build import load_inference_source
from ultralytics.data.utils import check_det_dataset
from ultralytics.utils import (
ARM64,
ASSETS,
ASSETS_URL,
DEFAULT_CFG,
DEFAULT_CFG_PATH,
IS_JETSON,
IS_RASPBERRYPI,
LINUX,
LOGGER,
ONLINE,
ROOT,
WEIGHTS_DIR,
WINDOWS,
YAML,
checks,
is_github_action_running,
)
from ultralytics.utils.downloads import download
from ultralytics.utils.torch_utils import TORCH_1_11, TORCH_1_13
def test_model_forward():
"""Test the forward pass of the YOLO model."""
model = YOLO(CFG)
model(source=None, imgsz=32, augment=True) # also test no source and augment
def test_model_methods():
"""Test various methods and properties of the YOLO model to ensure correct functionality."""
model = YOLO(MODEL)
# Model methods
model.info(verbose=True, detailed=True)
model = model.reset_weights()
model = model.load(MODEL)
model.to("cpu")
model.fuse()
model.clear_callback("on_train_start")
model.reset_callbacks()
# Model properties
_ = model.names
_ = model.device
_ = model.transforms
_ = model.task_map
def test_model_profile():
"""Test profiling of the YOLO model with `profile=True` to assess performance and resource usage."""
from ultralytics.nn.tasks import DetectionModel
model = DetectionModel() # build model
im = torch.randn(1, 3, 64, 64) # requires min imgsz=64
_ = model.predict(im, profile=True)
def test_predict_txt(tmp_path):
"""Test YOLO predictions with file, directory, and pattern sources listed in a text file."""
file = tmp_path / "sources_multi_row.txt"
with open(file, "w") as f:
for src in SOURCES_LIST:
f.write(f"{src}\n")
results = YOLO(MODEL)(source=file, imgsz=32)
assert len(results) == 7 # 1 + 2 + 2 + 2 = 7 images
@pytest.mark.skipif(True, reason="disabled for testing")
def test_predict_csv_multi_row(tmp_path):
"""Test YOLO predictions with sources listed in multiple rows of a CSV file."""
file = tmp_path / "sources_multi_row.csv"
with open(file, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(["source"])
writer.writerows([[src] for src in SOURCES_LIST])
results = YOLO(MODEL)(source=file, imgsz=32)
assert len(results) == 7 # 1 + 2 + 2 + 2 = 7 images
@pytest.mark.skipif(True, reason="disabled for testing")
def test_predict_csv_single_row(tmp_path):
"""Test YOLO predictions with sources listed in a single row of a CSV file."""
file = tmp_path / "sources_single_row.csv"
with open(file, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(SOURCES_LIST)
results = YOLO(MODEL)(source=file, imgsz=32)
assert len(results) == 7 # 1 + 2 + 2 + 2 = 7 images
@pytest.mark.parametrize("model_name", MODELS)
def test_predict_img(model_name):
"""Test YOLO model predictions on various image input types and sources, including online images."""
channels = 1 if model_name == "yolo11n-grayscale.pt" else 3
model = YOLO(WEIGHTS_DIR / model_name)
im = cv2.imread(str(SOURCE), flags=cv2.IMREAD_GRAYSCALE if channels == 1 else cv2.IMREAD_COLOR) # uint8 NumPy array
assert len(model(source=Image.open(SOURCE), save=True, verbose=True, imgsz=32)) == 1 # PIL
assert len(model(source=im, save=True, save_txt=True, imgsz=32)) == 1 # ndarray
assert len(model(torch.rand((2, channels, 32, 32)), imgsz=32)) == 2 # batch-size 2 Tensor, FP32 0.0-1.0 RGB order
assert len(model(source=[im, im], save=True, save_txt=True, imgsz=32)) == 2 # batch
assert len(list(model(source=[im, im], save=True, stream=True, imgsz=32))) == 2 # stream
assert len(model(torch.zeros(320, 640, channels).numpy().astype(np.uint8), imgsz=32)) == 1 # tensor to numpy
batch = [
str(SOURCE), # filename
Path(SOURCE), # Path
f"{ASSETS_URL}/zidane.jpg?token=123" if ONLINE else SOURCE, # URI
im, # OpenCV
Image.open(SOURCE), # PIL
np.zeros((320, 640, channels), dtype=np.uint8), # numpy
]
assert len(model(batch, imgsz=32, classes=0)) == len(batch) # multiple sources in a batch
@pytest.mark.parametrize("model", MODELS)
def test_predict_visualize(model):
"""Test model prediction methods with 'visualize=True' to generate prediction visualizations."""
YOLO(WEIGHTS_DIR / model)(SOURCE, imgsz=32, visualize=True)
def test_predict_gray_and_4ch(tmp_path):
"""Test YOLO prediction on SOURCE converted to grayscale and 4-channel images with various filenames."""
im = Image.open(SOURCE)
source_grayscale = tmp_path / "grayscale.jpg"
source_rgba = tmp_path / "4ch.png"
source_non_utf = tmp_path / "non_UTF_测试文件_tést_image.jpg"
source_spaces = tmp_path / "image with spaces.jpg"
im.convert("L").save(source_grayscale) # grayscale
im.convert("RGBA").save(source_rgba) # 4-ch PNG with alpha
im.save(source_non_utf) # non-UTF characters in filename
im.save(source_spaces) # spaces in filename
# Inference
model = YOLO(MODEL)
for f in source_rgba, source_grayscale, source_non_utf, source_spaces:
for source in Image.open(f), cv2.imread(str(f)), f:
results = model(source, save=True, verbose=True, imgsz=32)
assert len(results) == 1 # verify that an image was run
f.unlink() # cleanup
@pytest.mark.slow
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_predict_all_image_formats():
"""Predict on all 12 image formats (AVIF, BMP, DNG, HEIC, JP2, JPEG, JPG, MPO, PNG, TIF, TIFF, WebP)."""
# Download dataset if needed
data = check_det_dataset("coco12-formats.yaml")
dataset_path = Path(data["path"])
# Collect all images from train and val
expected = {"avif", "bmp", "dng", "heic", "jp2", "jpeg", "jpg", "mpo", "png", "tif", "tiff", "webp"}
images = [im for im in (dataset_path / "images" / "train").glob("*.*") if im.suffix.lower().lstrip(".") in expected]
images += [im for im in (dataset_path / "images" / "val").glob("*.*") if im.suffix.lower().lstrip(".") in expected]
assert len(images) == 12, f"Expected 12 images, found {len(images)}"
# Verify all format extensions are represented
extensions = {img.suffix.lower().lstrip(".") for img in images}
assert extensions == expected, f"Missing formats: {expected - extensions}"
# Run inference on all images
model = YOLO(MODEL)
results = model(images, imgsz=32)
assert len(results) == 12, f"Expected 12 results, got {len(results)}"
@pytest.mark.slow
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
@pytest.mark.skipif(is_github_action_running(), reason="No auth https://github.com/JuanBindez/pytubefix/issues/166")
def test_youtube():
"""Test YOLO model on a YouTube video stream, handling potential network-related errors."""
model = YOLO(MODEL)
try:
model.predict("https://youtu.be/G17sBkb38XQ", imgsz=96, save=True)
# Handle internet connection errors and 'urllib.error.HTTPError: HTTP Error 429: Too Many Requests'
except (urllib.error.HTTPError, ConnectionError) as e:
LOGGER.error(f"YouTube Test Error: {e}")
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
@pytest.mark.parametrize("model", MODELS)
def test_track_stream(model, tmp_path):
"""Test streaming tracking on a short 10 frame video using ByteTrack tracker and different GMC methods.
Note imgsz=160 required for tracking for higher confidence and better matches.
"""
if model == "yolo26n-cls.pt": # classification model not supported for tracking
return
video_url = f"{ASSETS_URL}/decelera_portrait_min.mov"
model = YOLO(model)
model.track(video_url, imgsz=160, tracker="bytetrack.yaml")
model.track(video_url, imgsz=160, tracker="botsort.yaml", save_frames=True) # test frame saving also
# Test Global Motion Compensation (GMC) methods and ReID
for gmc, reidm in zip(["orb", "sift", "ecc"], ["auto", "auto", "yolo26n-cls.pt"]):
default_args = YAML.load(ROOT / "cfg/trackers/botsort.yaml")
custom_yaml = tmp_path / f"botsort-{gmc}.yaml"
YAML.save(custom_yaml, {**default_args, "gmc_method": gmc, "with_reid": True, "model": reidm})
model.track(video_url, imgsz=160, tracker=custom_yaml)
@pytest.mark.parametrize("task,weight,data", TASK_MODEL_DATA)
def test_val(task: str, weight: str, data: str) -> None:
"""Test the validation mode of the YOLO model."""
model = YOLO(weight)
for plots in {True, False}: # Test both cases i.e. plots=True and plots=False
metrics = model.val(data=data, imgsz=32, plots=plots)
metrics.to_df()
metrics.to_csv()
metrics.to_json()
# Tests for confusion matrix export
metrics.confusion_matrix.to_df()
metrics.confusion_matrix.to_csv()
metrics.confusion_matrix.to_json()
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
@pytest.mark.skipif(IS_JETSON or IS_RASPBERRYPI, reason="Edge devices not intended for training")
def test_train_scratch():
"""Test training the YOLO model from scratch on 12 different image types in the COCO12-Formats dataset."""
model = YOLO(CFG)
model.train(data="coco12-formats.yaml", epochs=2, imgsz=32, cache="disk", batch=-1, close_mosaic=1, name="model")
model(SOURCE)
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_train_ndjson():
"""Test training the YOLO model using NDJSON format dataset."""
model = YOLO(WEIGHTS_DIR / "yolo26n.pt")
model.train(data=f"{ASSETS_URL}/coco8-ndjson.ndjson", epochs=1, imgsz=32)
@pytest.mark.parametrize("scls", [False, True])
def test_train_pretrained(scls):
"""Test training of the YOLO model starting from a pre-trained checkpoint."""
model = YOLO(WEIGHTS_DIR / "yolo26n-seg.pt")
model.train(
data="coco8-seg.yaml", epochs=1, imgsz=32, cache="ram", copy_paste=0.5, mixup=0.5, name=0, single_cls=scls
)
model(SOURCE)
def test_all_model_yamls():
"""Test YOLO model creation for all available YAML configurations in the `cfg/models` directory."""
for m in (ROOT / "cfg" / "models").rglob("*.yaml"):
if "rtdetr" in m.name:
if TORCH_1_11:
_ = RTDETR(m.name)(SOURCE, imgsz=640) # must be 640
else:
YOLO(m.name)
@pytest.mark.skipif(WINDOWS, reason="Windows slow CI export bug https://github.com/ultralytics/ultralytics/pull/16003")
def test_workflow():
"""Test the complete workflow including training, validation, prediction, and exporting."""
model = YOLO(MODEL)
model.train(data="coco8.yaml", epochs=1, imgsz=32, optimizer="SGD")
model.val(imgsz=32)
model.predict(SOURCE, imgsz=32)
model.export(format="torchscript") # WARNING: Windows slow CI export bug
def test_predict_callback_and_setup():
"""Test callback functionality during YOLO prediction setup and execution."""
def on_predict_batch_end(predictor):
"""Callback function that handles operations at the end of a prediction batch."""
path, im0s, _ = predictor.batch
im0s = im0s if isinstance(im0s, list) else [im0s]
bs = [predictor.dataset.bs for _ in range(len(path))]
predictor.results = zip(predictor.results, im0s, bs) # results is list[batch_size]
model = YOLO(MODEL)
model.add_callback("on_predict_batch_end", on_predict_batch_end)
dataset = load_inference_source(source=SOURCE)
bs = dataset.bs # access predictor properties
results = model.predict(dataset, stream=True, imgsz=160) # source already setup
for r, im0, bs in results:
print("test_callback", im0.shape)
print("test_callback", bs)
boxes = r.boxes # Boxes object for bbox outputs
print(boxes)
@pytest.mark.parametrize("model", MODELS)
def test_results(model: str, tmp_path):
"""Test YOLO model results processing and output in various formats."""
im = f"{ASSETS_URL}/boats.jpg" if model == "yolo26n-obb.pt" else SOURCE
results = YOLO(WEIGHTS_DIR / model)([im, im], imgsz=160)
for r in results:
assert len(r), f"'{model}' results should not be empty!"
r = r.cpu().numpy()
print(r, len(r), r.path) # print numpy attributes
r = r.to(device="cpu", dtype=torch.float32)
r.save_txt(txt_file=tmp_path / "runs/tests/label.txt", save_conf=True)
r.save_crop(save_dir=tmp_path / "runs/tests/crops/")
r.to_df(decimals=3) # Align to_ methods: https://docs.ultralytics.com/modes/predict/#working-with-results
r.to_csv()
r.to_json(normalize=True)
r.plot(pil=True, save=True, filename=tmp_path / "results_plot_save.jpg")
r.plot(conf=True, boxes=True)
print(r, len(r), r.path) # print after methods
def test_labels_and_crops():
"""Test output from prediction args for saving YOLO detection labels and crops."""
imgs = [SOURCE, ASSETS / "zidane.jpg"]
results = YOLO(WEIGHTS_DIR / "yolo26n.pt")(imgs, imgsz=320, save_txt=True, save_crop=True)
save_path = Path(results[0].save_dir)
for r in results:
im_name = Path(r.path).stem
cls_idxs = r.boxes.cls.int().tolist()
# Check that detections are made (at least 2 detections per image expected)
assert len(cls_idxs) >= 2, f"Expected at least 2 detections, got {len(cls_idxs)}"
# Check label path
labels = save_path / f"labels/{im_name}.txt"
assert labels.exists()
# Check detections match label count
assert len(r.boxes.data) == len([line for line in labels.read_text().splitlines() if line])
# Check crops path and files
crop_dirs = list((save_path / "crops").iterdir())
crop_files = [f for p in crop_dirs for f in p.glob("*")]
# Crop directories match detections
assert all(r.names.get(c) in {d.name for d in crop_dirs} for c in cls_idxs)
# Same number of crops as detections
assert len([f for f in crop_files if im_name in f.name]) == len(r.boxes.data)
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_data_utils(tmp_path):
"""Test data utility functions including dataset stats, auto-splitting, and zip archiving."""
from ultralytics.data.split import autosplit
from ultralytics.data.utils import HUBDatasetStats
from ultralytics.utils.downloads import zip_directory
# from ultralytics.utils.files import WorkingDirectory
# with WorkingDirectory(ROOT.parent / 'tests'):
for task in TASKS:
file = Path(TASK2DATA[task]).with_suffix(".zip") # i.e. coco8.zip
download(f"https://github.com/ultralytics/hub/raw/main/example_datasets/{file}", unzip=False, dir=tmp_path)
stats = HUBDatasetStats(tmp_path / file, task=task)
stats.get_json(save=True)
stats.process_images()
autosplit(tmp_path / "coco8")
zip_directory(tmp_path / "coco8/images/val") # zip
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_data_converter(tmp_path):
"""Test dataset conversion functions from COCO to YOLO format and class mappings."""
from ultralytics.data.converter import coco80_to_coco91_class, convert_coco
download(f"{ASSETS_URL}/instances_val2017.json", dir=tmp_path)
convert_coco(
labels_dir=tmp_path, save_dir=tmp_path / "yolo_labels", use_segments=True, use_keypoints=False, cls91to80=True
)
coco80_to_coco91_class()
def test_data_annotator(tmp_path):
"""Test automatic annotation of data using detection and segmentation models."""
from ultralytics.data.annotator import auto_annotate
auto_annotate(
ASSETS,
det_model=WEIGHTS_DIR / "yolo26n.pt",
sam_model=WEIGHTS_DIR / "mobile_sam.pt",
output_dir=tmp_path / "auto_annotate_labels",
)
def test_events():
"""Test event sending functionality."""
from ultralytics.utils.events import Events
events = Events()
events.enabled = True
cfg = copy(DEFAULT_CFG) # does not require deepcopy
cfg.mode = "test"
events(cfg)
def test_cfg_init():
"""Test configuration initialization utilities from the 'ultralytics.cfg' module."""
from ultralytics.cfg import check_dict_alignment, copy_default_cfg, smart_value
with contextlib.suppress(SyntaxError):
check_dict_alignment({"a": 1}, {"b": 2})
copy_default_cfg()
(Path.cwd() / DEFAULT_CFG_PATH.name.replace(".yaml", "_copy.yaml")).unlink(missing_ok=False)
# Test smart_value() with comprehensive cases
# Test None conversion
assert smart_value("none") is None
assert smart_value("None") is None
assert smart_value("NONE") is None
# Test boolean conversion
assert smart_value("true") is True
assert smart_value("True") is True
assert smart_value("TRUE") is True
assert smart_value("false") is False
assert smart_value("False") is False
assert smart_value("FALSE") is False
# Test numeric conversion (ast.literal_eval)
assert smart_value("42") == 42
assert smart_value("-42") == -42
assert smart_value("3.14") == 3.14
assert smart_value("-3.14") == -3.14
assert smart_value("1e-3") == 0.001
# Test list/tuple conversion (ast.literal_eval)
assert smart_value("[1, 2, 3]") == [1, 2, 3]
assert smart_value("(1, 2, 3)") == (1, 2, 3)
assert smart_value("[640, 640]") == [640, 640]
# Test dict conversion (ast.literal_eval)
assert smart_value("{'a': 1, 'b': 2}") == {"a": 1, "b": 2}
# Test string fallback (when ast.literal_eval fails)
assert smart_value("some_string") == "some_string"
assert smart_value("path/to/file") == "path/to/file"
assert smart_value("hello world") == "hello world"
# Test that code injection is prevented (ast.literal_eval safety)
# These should return strings, not execute code
assert smart_value("__import__('os').system('ls')") == "__import__('os').system('ls')"
assert smart_value("eval('1+1')") == "eval('1+1')"
assert smart_value("exec('x=1')") == "exec('x=1')"
def test_utils_init():
"""Test initialization utilities in the Ultralytics library."""
from ultralytics.utils import get_ubuntu_version, is_github_action_running
get_ubuntu_version()
is_github_action_running()
def test_utils_checks():
"""Test various utility checks for filenames, requirements, image sizes, display capabilities, and versions."""
checks.check_yolov5u_filename("yolov5n.pt")
checks.check_requirements("numpy") # check requirements.txt
checks.check_imgsz([600, 600], max_dim=1)
checks.check_imshow(warn=True)
checks.check_version("ultralytics", "8.0.0")
checks.print_args()
@pytest.mark.skipif(WINDOWS, reason="Windows profiling is extremely slow (cause unknown)")
def test_utils_benchmarks():
"""Benchmark model performance using 'ProfileModels' from 'ultralytics.utils.benchmarks'."""
from ultralytics.utils.benchmarks import ProfileModels
ProfileModels(["yolo26n.yaml"], imgsz=32, min_time=1, num_timed_runs=3, num_warmup_runs=1).run()
def test_utils_torchutils():
"""Test Torch utility functions including profiling and FLOP calculations."""
from ultralytics.nn.modules.conv import Conv
from ultralytics.utils.torch_utils import get_flops_with_torch_profiler, profile_ops, time_sync
x = torch.randn(1, 64, 20, 20)
m = Conv(64, 64, k=1, s=2)
profile_ops(x, [m], n=3)
get_flops_with_torch_profiler(m)
time_sync()
def test_utils_ops():
"""Test utility operations for coordinate transformations and normalizations."""
from ultralytics.utils.ops import (
ltwh2xywh,
ltwh2xyxy,
make_divisible,
xywh2ltwh,
xywh2xyxy,
xywhn2xyxy,
xywhr2xyxyxyxy,
xyxy2ltwh,
xyxy2xywh,
xyxy2xywhn,
xyxyxyxy2xywhr,
)
make_divisible(17, torch.tensor([8]))
boxes = torch.rand(10, 4) # xywh
torch.allclose(boxes, xyxy2xywh(xywh2xyxy(boxes)))
torch.allclose(boxes, xyxy2xywhn(xywhn2xyxy(boxes)))
torch.allclose(boxes, ltwh2xywh(xywh2ltwh(boxes)))
torch.allclose(boxes, xyxy2ltwh(ltwh2xyxy(boxes)))
boxes = torch.rand(10, 5) # xywhr for OBB
boxes[:, 4] = torch.randn(10) * 30
torch.allclose(boxes, xyxyxyxy2xywhr(xywhr2xyxyxyxy(boxes)), rtol=1e-3)
def test_utils_files(tmp_path):
"""Test file handling utilities including file age, date, and paths with spaces."""
from ultralytics.utils.files import file_age, file_date, get_latest_run, spaces_in_path
file_age(SOURCE)
file_date(SOURCE)
get_latest_run(ROOT / "runs")
path = tmp_path / "path/with spaces"
path.mkdir(parents=True, exist_ok=True)
with spaces_in_path(path) as new_path:
print(new_path)
@pytest.mark.slow
def test_utils_patches_torch_save(tmp_path):
"""Test torch_save backoff when _torch_save raises RuntimeError."""
from unittest.mock import MagicMock, patch
from ultralytics.utils.patches import torch_save
mock = MagicMock(side_effect=RuntimeError)
with patch("ultralytics.utils.patches._torch_save", new=mock):
with pytest.raises(RuntimeError):
torch_save(torch.zeros(1), tmp_path / "test.pt")
assert mock.call_count == 4, "torch_save was not attempted the expected number of times"
def test_nn_modules_conv():
"""Test Convolutional Neural Network modules including CBAM, Conv2, and ConvTranspose."""
from ultralytics.nn.modules.conv import CBAM, Conv2, ConvTranspose, DWConvTranspose2d, Focus
c1, c2 = 8, 16 # input and output channels
x = torch.zeros(4, c1, 10, 10) # BCHW
# Run all modules not otherwise covered in tests
DWConvTranspose2d(c1, c2)(x)
ConvTranspose(c1, c2)(x)
Focus(c1, c2)(x)
CBAM(c1)(x)
# Fuse ops
m = Conv2(c1, c2)
m.fuse_convs()
m(x)
def test_nn_modules_block():
"""Test various neural network block modules."""
from ultralytics.nn.modules.block import C1, C3TR, BottleneckCSP, C3Ghost, C3x
c1, c2 = 8, 16 # input and output channels
x = torch.zeros(4, c1, 10, 10) # BCHW
# Run all modules not otherwise covered in tests
C1(c1, c2)(x)
C3x(c1, c2)(x)
C3TR(c1, c2)(x)
C3Ghost(c1, c2)(x)
BottleneckCSP(c1, c2)(x)
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_hub():
"""Test Ultralytics HUB functionalities."""
from ultralytics.hub import export_fmts_hub, logout
from ultralytics.hub.utils import smart_request
export_fmts_hub()
logout()
smart_request("GET", "https://github.com", progress=True)
@pytest.fixture
def image():
"""Load and return an image from a predefined source (OpenCV BGR)."""
return cv2.imread(str(SOURCE))
@pytest.mark.parametrize(
"auto_augment, erasing, force_color_jitter",
[
(None, 0.0, False),
("randaugment", 0.5, True),
("augmix", 0.2, False),
("autoaugment", 0.0, True),
],
)
def test_classify_transforms_train(image, auto_augment, erasing, force_color_jitter):
"""Test classification transforms during training with various augmentations."""
from ultralytics.data.augment import classify_augmentations
transform = classify_augmentations(
size=224,
mean=(0.5, 0.5, 0.5),
std=(0.5, 0.5, 0.5),
scale=(0.08, 1.0),
ratio=(3.0 / 4.0, 4.0 / 3.0),
hflip=0.5,
vflip=0.5,
auto_augment=auto_augment,
hsv_h=0.015,
hsv_s=0.4,
hsv_v=0.4,
force_color_jitter=force_color_jitter,
erasing=erasing,
)
transformed_image = transform(Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)))
assert transformed_image.shape == (3, 224, 224)
assert torch.is_tensor(transformed_image)
assert transformed_image.dtype == torch.float32
@pytest.mark.slow
@pytest.mark.skipif(not ONLINE, reason="environment is offline")
def test_model_tune():
"""Tune YOLO model for performance improvement."""
YOLO("yolo26n-pose.pt").tune(data="coco8-pose.yaml", plots=False, imgsz=32, epochs=1, iterations=2, device="cpu")
YOLO("yolo26n-cls.pt").tune(data="imagenet10", plots=False, imgsz=32, epochs=1, iterations=2, device="cpu")
def test_model_embeddings():
"""Test YOLO model embeddings extraction functionality."""
model_detect = YOLO(MODEL)
model_segment = YOLO(WEIGHTS_DIR / "yolo26n-seg.pt")
for batch in [SOURCE], [SOURCE, SOURCE]: # test batch size 1 and 2
assert len(model_detect.embed(source=batch, imgsz=32)) == len(batch)
assert len(model_segment.embed(source=batch, imgsz=32)) == len(batch)
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOWorld with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(
checks.IS_PYTHON_3_8 and LINUX and ARM64,
reason="YOLOWorld with CLIP is not supported in Python 3.8 and aarch64 Linux",
)
def test_yolo_world():
"""Test YOLO world models with CLIP support."""
model = YOLO(WEIGHTS_DIR / "yolov8s-world.pt") # no YOLO11n-world model yet
model.set_classes(["tree", "window"])
model(SOURCE, conf=0.01)
model = YOLO(WEIGHTS_DIR / "yolov8s-worldv2.pt") # no YOLO11n-world model yet
# Training from a pretrained model. Eval is included at the final stage of training.
# Use dota8.yaml which has fewer categories to reduce the inference time of CLIP model
model.train(
data="dota8.yaml",
epochs=1,
imgsz=32,
cache="disk",
close_mosaic=1,
)
# test WorWorldTrainerFromScratch
from ultralytics.models.yolo.world.train_world import WorldTrainerFromScratch
model = YOLO("yolov8s-worldv2.yaml") # no YOLO11n-world model yet
model.train(
data={"train": {"yolo_data": ["dota8.yaml"]}, "val": {"yolo_data": ["dota8.yaml"]}},
epochs=1,
imgsz=32,
cache="disk",
close_mosaic=1,
trainer=WorldTrainerFromScratch,
)
@pytest.mark.skipif(not TORCH_1_13, reason="YOLOE with CLIP requires torch>=1.13")
@pytest.mark.skipif(checks.IS_PYTHON_3_12, reason="YOLOE with CLIP is not supported in Python 3.12")
@pytest.mark.skipif(
checks.IS_PYTHON_3_8 and LINUX and ARM64,
reason="YOLOE with CLIP is not supported in Python 3.8 and aarch64 Linux",
)
def test_yoloe(tmp_path):
"""Test YOLOE models with MobileCLIP support."""
# Predict
# text-prompts
model = YOLO(WEIGHTS_DIR / "yoloe-11s-seg.pt")
model.set_classes(["person", "bus"])
model(SOURCE, conf=0.01)
from ultralytics import YOLOE
from ultralytics.models.yolo.yoloe import YOLOEVPSegPredictor
# visual-prompts
visuals = dict(
bboxes=np.array([[221.52, 405.8, 344.98, 857.54], [120, 425, 160, 445]]),
cls=np.array([0, 1]),
)
model.predict(
SOURCE,
visual_prompts=visuals,
predictor=YOLOEVPSegPredictor,
)
# Val
model = YOLOE(WEIGHTS_DIR / "yoloe-11s-seg.pt")
# text prompts
model.val(data="coco128-seg.yaml", imgsz=32)
# visual prompts
model.val(data="coco128-seg.yaml", load_vp=True, imgsz=32)
# Train, fine-tune
from ultralytics.models.yolo.yoloe import YOLOEPESegTrainer, YOLOESegTrainerFromScratch
model = YOLOE("yoloe-11s-seg.pt")
model.train(
data="coco128-seg.yaml",
epochs=1,
close_mosaic=1,
trainer=YOLOEPESegTrainer,
imgsz=32,
)
# Train, from scratch
data_dict = dict(train=dict(yolo_data=["coco128-seg.yaml"]), val=dict(yolo_data=["coco128-seg.yaml"]))
data_yaml = tmp_path / "yoloe-data.yaml"
YAML.save(data=data_dict, file=data_yaml)
for data in [data_dict, data_yaml]:
model = YOLOE("yoloe-11s-seg.yaml")
model.train(
data=data,
epochs=1,
close_mosaic=1,
trainer=YOLOESegTrainerFromScratch,
imgsz=32,
)
# prompt-free
# predict
model = YOLOE(WEIGHTS_DIR / "yoloe-11s-seg-pf.pt")
model.predict(SOURCE)
# val
model = YOLOE("yoloe-11s-seg.pt") # or select yoloe-m/l-seg.pt for different sizes
model.val(data="coco128-seg.yaml", imgsz=32)
def test_yolov10():
"""Test YOLOv10 model training, validation, and prediction functionality."""
model = YOLO("yolov10n.yaml")
# train/val/predict
model.train(data="coco8.yaml", epochs=1, imgsz=32, close_mosaic=1, cache="disk")
model.val(data="coco8.yaml", imgsz=32)
model.predict(imgsz=32, save_txt=True, save_crop=True, augment=True)
model(SOURCE)
def test_multichannel():
"""Test YOLO model multi-channel training, validation, and prediction functionality."""
model = YOLO("yolo26n.pt")
model.train(data="coco8-multispectral.yaml", epochs=1, imgsz=32, close_mosaic=1, cache="disk")
model.val(data="coco8-multispectral.yaml")
im = np.zeros((32, 32, 10), dtype=np.uint8)
model.predict(source=im, imgsz=32, save_txt=True, save_crop=True, augment=True)
model.export(format="onnx")
@pytest.mark.parametrize("task,model,data", TASK_MODEL_DATA)
def test_grayscale(task: str, model: str, data: str, tmp_path) -> None:
"""Test YOLO model grayscale training, validation, and prediction functionality."""
if task == "classify": # not support grayscale classification yet
return
grayscale_data = tmp_path / f"{Path(data).stem}-grayscale.yaml"
data = check_det_dataset(data)
data["channels"] = 1 # add additional channels key for grayscale
YAML.save(data=data, file=grayscale_data)
# remove npy files in train/val splits if exists, might be created by previous tests
for split in {"train", "val"}:
for npy_file in (Path(data["path"]) / data[split]).glob("*.npy"):
npy_file.unlink()
model = YOLO(model)
model.train(data=grayscale_data, epochs=1, imgsz=32, close_mosaic=1, cache="disk")
# remove npy files in train/val splits if exists, avoiding interference with other tests
for split in {"train", "val"}:
for npy_file in (Path(data["path"]) / data[split]).glob("*.npy"):
npy_file.unlink()
model.val(data=grayscale_data)
im = np.zeros((32, 32, 1), dtype=np.uint8)
model.predict(source=im, imgsz=32, save_txt=True, save_crop=True, augment=True)
export_model = model.export(format="onnx")
model = YOLO(export_model, task=task)
model.predict(source=im, imgsz=32)

371
tests/test_solutions.py Executable file
View File

@@ -0,0 +1,371 @@
# Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
# Tests Ultralytics Solutions: https://docs.ultralytics.com/solutions/,
# Includes all solutions except DistanceCalculation and the Security Alarm System.
import os
from unittest.mock import patch
import cv2
import numpy as np
import pytest
from tests import MODEL
from ultralytics import solutions
from ultralytics.utils import ASSETS_URL, IS_RASPBERRYPI, TORCH_VERSION, checks
from ultralytics.utils.downloads import safe_download
from ultralytics.utils.torch_utils import TORCH_2_4
# Predefined argument values
SHOW = False
DEMO_VIDEO = "solutions_ci_demo.mp4" # for all the solutions, except workout, object cropping and parking management
CROP_VIDEO = "decelera_landscape_min.mov" # for object cropping solution
POSE_VIDEO = "solution_ci_pose_demo.mp4" # only for workouts monitoring solution
PARKING_VIDEO = "solution_ci_parking_demo.mp4" # only for parking management solution
PARKING_AREAS_JSON = "solution_ci_parking_areas.json" # only for parking management solution
PARKING_MODEL = "solutions_ci_parking_model.pt" # only for parking management solution
VERTICAL_VIDEO = "solution_vertical_demo.mp4" # only for vertical line counting
REGION = [(10, 200), (540, 200), (540, 180), (10, 180)] # for object counting, speed estimation and queue management
HORIZONTAL_LINE = [(10, 200), (540, 200)] # for object counting
VERTICAL_LINE = [(320, 0), (320, 400)] # for object counting
def process_video(solution, video_path: str, needs_frame_count: bool = False):
"""Process video with solution, feeding frames and optional frame count to the solution instance."""
cap = cv2.VideoCapture(video_path)
assert cap.isOpened(), f"Error reading video file {video_path}"
frame_count = 0
while cap.isOpened():
success, im0 = cap.read()
if not success:
break
frame_count += 1
im_copy = im0.copy()
args = [im_copy, frame_count] if needs_frame_count else [im_copy]
_ = solution(*args)
cap.release()
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Disabled for testing due to --slow test errors after YOLOE PR.")
@pytest.mark.parametrize(
"name, solution_class, needs_frame_count, video, kwargs",
[
(
"ObjectCounter",
solutions.ObjectCounter,
False,
DEMO_VIDEO,
{"region": REGION, "model": MODEL, "show": SHOW},
),
(
"ObjectCounter",
solutions.ObjectCounter,
False,
DEMO_VIDEO,
{"region": HORIZONTAL_LINE, "model": MODEL, "show": SHOW},
),
(
"ObjectCounterVertical",
solutions.ObjectCounter,
False,
DEMO_VIDEO,
{"region": VERTICAL_LINE, "model": MODEL, "show": SHOW},
),
(
"ObjectCounterwithOBB",
solutions.ObjectCounter,
False,
DEMO_VIDEO,
{"region": REGION, "model": "yolo26n-obb.pt", "show": SHOW},
),
(
"Heatmap",
solutions.Heatmap,
False,
DEMO_VIDEO,
{"colormap": cv2.COLORMAP_PARULA, "model": MODEL, "show": SHOW, "region": None},
),
(
"HeatmapWithRegion",
solutions.Heatmap,
False,
DEMO_VIDEO,
{"colormap": cv2.COLORMAP_PARULA, "region": REGION, "model": MODEL, "show": SHOW},
),
(
"SpeedEstimator",
solutions.SpeedEstimator,
False,
DEMO_VIDEO,
{"region": REGION, "model": MODEL, "show": SHOW},
),
(
"QueueManager",
solutions.QueueManager,
False,
DEMO_VIDEO,
{"region": REGION, "model": MODEL, "show": SHOW},
),
(
"LineAnalytics",
solutions.Analytics,
True,
DEMO_VIDEO,
{"analytics_type": "line", "model": MODEL, "show": SHOW, "figsize": (6.4, 3.2)},
),
(
"PieAnalytics",
solutions.Analytics,
True,
DEMO_VIDEO,
{"analytics_type": "pie", "model": MODEL, "show": SHOW, "figsize": (6.4, 3.2)},
),
(
"BarAnalytics",
solutions.Analytics,
True,
DEMO_VIDEO,
{"analytics_type": "bar", "model": MODEL, "show": SHOW, "figsize": (6.4, 3.2)},
),
(
"AreaAnalytics",
solutions.Analytics,
True,
DEMO_VIDEO,
{"analytics_type": "area", "model": MODEL, "show": SHOW, "figsize": (6.4, 3.2)},
),
("TrackZone", solutions.TrackZone, False, DEMO_VIDEO, {"region": REGION, "model": MODEL, "show": SHOW}),
(
"ObjectCropper",
solutions.ObjectCropper,
False,
CROP_VIDEO,
{"temp_crop_dir": "cropped-detections", "model": MODEL, "show": SHOW},
),
(
"ObjectBlurrer",
solutions.ObjectBlurrer,
False,
DEMO_VIDEO,
{"blur_ratio": 0.02, "model": MODEL, "show": SHOW},
),
(
"InstanceSegmentation",
solutions.InstanceSegmentation,
False,
DEMO_VIDEO,
{"model": "yolo26n-seg.pt", "show": SHOW},
),
("VisionEye", solutions.VisionEye, False, DEMO_VIDEO, {"model": MODEL, "show": SHOW}),
(
"RegionCounter",
solutions.RegionCounter,
False,
DEMO_VIDEO,
{"region": REGION, "model": MODEL, "show": SHOW},
),
("AIGym", solutions.AIGym, False, POSE_VIDEO, {"kpts": [6, 8, 10], "show": SHOW}),
(
"ParkingManager",
solutions.ParkingManagement,
False,
PARKING_VIDEO,
{"temp_model": str(PARKING_MODEL), "show": SHOW, "temp_json_file": str(PARKING_AREAS_JSON)},
),
(
"StreamlitInference",
solutions.Inference,
False,
None, # streamlit application doesn't require video file
{}, # streamlit application doesn't accept arguments
),
],
)
def test_solution(name, solution_class, needs_frame_count, video, kwargs, tmp_path):
"""Test individual Ultralytics solution with video processing and parameter validation."""
if video:
if name != "ObjectCounterVertical":
safe_download(url=f"{ASSETS_URL}/{video}", dir=tmp_path)
else:
safe_download(url=f"{ASSETS_URL}/{VERTICAL_VIDEO}", dir=tmp_path)
if name == "ParkingManager":
safe_download(url=f"{ASSETS_URL}/{PARKING_AREAS_JSON}", dir=tmp_path)
safe_download(url=f"{ASSETS_URL}/{PARKING_MODEL}", dir=tmp_path)
elif name == "StreamlitInference":
if checks.check_imshow(): # do not merge with elif above
solution_class(**kwargs).inference() # requires interactive GUI environment
return
# Update kwargs to use tmp_path
kwargs_updated = {}
for key in kwargs:
if key.startswith("temp_"):
kwargs_updated[key.replace("temp_", "")] = str(tmp_path / kwargs[key])
else:
kwargs_updated[key] = kwargs[key]
video = VERTICAL_VIDEO if name == "ObjectCounterVertical" else video
process_video(
solution=solution_class(**kwargs_updated),
video_path=str(tmp_path / video),
needs_frame_count=needs_frame_count,
)
def test_left_click_selection():
"""Test distance calculation left click selection functionality."""
dc = solutions.DistanceCalculation()
dc.boxes, dc.track_ids = [[10, 10, 50, 50]], [1]
dc.mouse_event_for_distance(cv2.EVENT_LBUTTONDOWN, 30, 30, None, None)
assert 1 in dc.selected_boxes
def test_right_click_reset():
"""Test distance calculation right click reset functionality."""
dc = solutions.DistanceCalculation()
dc.selected_boxes, dc.left_mouse_count = {1: [10, 10, 50, 50]}, 1
dc.mouse_event_for_distance(cv2.EVENT_RBUTTONDOWN, 0, 0, None, None)
assert not dc.selected_boxes
assert dc.left_mouse_count == 0
def test_parking_json_none():
"""Test that ParkingManagement handles missing JSON gracefully."""
im0 = np.zeros((640, 480, 3), dtype=np.uint8)
try:
parkingmanager = solutions.ParkingManagement(json_path=None)
parkingmanager(im0)
except ValueError:
pytest.skip("Skipping test due to missing JSON.")
def test_analytics_graph_not_supported():
"""Test that unsupported analytics type raises ValueError."""
try:
analytics = solutions.Analytics(analytics_type="test") # 'test' is unsupported
analytics.process(im0=np.zeros((640, 480, 3), dtype=np.uint8), frame_number=0)
assert False, "Expected ValueError for unsupported chart type"
except ValueError as e:
assert "Unsupported analytics_type" in str(e)
def test_area_chart_padding():
"""Test area chart graph update with dynamic class padding logic."""
analytics = solutions.Analytics(analytics_type="area")
analytics.update_graph(frame_number=1, count_dict={"car": 2}, plot="area")
plot_im = analytics.update_graph(frame_number=2, count_dict={"car": 3, "person": 1}, plot="area")
assert plot_im is not None
def test_config_update_method_with_invalid_argument():
"""Test that update() raises ValueError for invalid config keys."""
obj = solutions.config.SolutionConfig()
try:
obj.update(invalid_key=123)
assert False, "Expected ValueError for invalid update argument"
except ValueError as e:
assert "is not a valid solution argument" in str(e)
def test_plot_with_no_masks():
"""Test that instance segmentation handles cases with no masks."""
im0 = np.zeros((640, 480, 3), dtype=np.uint8)
isegment = solutions.InstanceSegmentation(model="yolo26n-seg.pt")
results = isegment(im0)
assert results.plot_im is not None
def test_streamlit_handle_video_upload_creates_file():
"""Test Streamlit video upload logic saves file correctly."""
import io
fake_file = io.BytesIO(b"fake video content")
fake_file.read = fake_file.getvalue
if fake_file is not None:
g = io.BytesIO(fake_file.read())
with open("ultralytics.mp4", "wb") as out:
out.write(g.read())
output_path = "ultralytics.mp4"
else:
output_path = None
assert output_path == "ultralytics.mp4"
assert os.path.exists("ultralytics.mp4")
with open("ultralytics.mp4", "rb") as f:
assert f.read() == b"fake video content"
os.remove("ultralytics.mp4")
@pytest.mark.skipif(not TORCH_2_4, reason=f"VisualAISearch requires torch>=2.4 (found torch=={TORCH_VERSION})")
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Disabled due to slow performance on Raspberry Pi.")
def test_similarity_search(tmp_path):
"""Test similarity search solution with sample images and text query."""
safe_download(f"{ASSETS_URL}/4-imgs-similaritysearch.zip", dir=tmp_path) # 4 dog images for testing in a zip file
searcher = solutions.VisualAISearch(data=str(tmp_path / "4-imgs-similaritysearch"))
_ = searcher("a dog sitting on a bench") # Returns the results in format "- img name | similarity score"
@pytest.mark.skipif(not TORCH_2_4, reason=f"VisualAISearch requires torch>=2.4 (found torch=={TORCH_VERSION})")
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Disabled due to slow performance on Raspberry Pi.")
def test_similarity_search_app_init():
"""Test SearchApp initializes with required attributes."""
app = solutions.SearchApp(device="cpu")
assert hasattr(app, "searcher")
assert hasattr(app, "run")
@pytest.mark.skipif(not TORCH_2_4, reason=f"VisualAISearch requires torch>=2.4 (found torch=={TORCH_VERSION})")
@pytest.mark.skipif(IS_RASPBERRYPI, reason="Disabled due to slow performance on Raspberry Pi.")
def test_similarity_search_complete(tmp_path):
"""Test VisualAISearch end-to-end with sample images and query."""
from PIL import Image
image_dir = tmp_path / "images"
os.makedirs(image_dir, exist_ok=True)
for i in range(2):
img = Image.fromarray(np.uint8(np.random.rand(224, 224, 3) * 255))
img.save(image_dir / f"test_image_{i}.jpg")
searcher = solutions.VisualAISearch(data=str(image_dir))
results = searcher("a red and white object")
assert results
def test_distance_calculation_process_method():
"""Test DistanceCalculation.process() computes distance between selected boxes."""
from ultralytics.solutions.solutions import SolutionResults
dc = solutions.DistanceCalculation()
dc.boxes, dc.track_ids, dc.clss, dc.confs = (
[[100, 100, 200, 200], [300, 300, 400, 400]],
[1, 2],
[0, 0],
[0.9, 0.95],
)
dc.selected_boxes = {1: dc.boxes[0], 2: dc.boxes[1]}
frame = np.zeros((480, 640, 3), dtype=np.uint8)
with patch.object(dc, "extract_tracks"), patch.object(dc, "display_output"), patch("cv2.setMouseCallback"):
result = dc.process(frame)
assert isinstance(result, SolutionResults)
assert result.total_tracks == 2
assert result.pixels_distance > 0
def test_object_crop_with_show_True():
"""Test ObjectCropper init with show=True to cover display warning."""
solutions.ObjectCropper(show=True)
def test_display_output_method():
"""Test that display_output triggers imshow, waitKey, and destroyAllWindows when enabled."""
counter = solutions.ObjectCounter(show=True)
counter.env_check = True
frame = np.zeros((100, 100, 3), dtype=np.uint8)
with patch("cv2.imshow") as mock_imshow, patch("cv2.waitKey", return_value=ord("q")) as mock_wait, patch(
"cv2.destroyAllWindows"
) as mock_destroy:
counter.display_output(frame)
mock_imshow.assert_called_once()
mock_wait.assert_called_once()
mock_destroy.assert_called_once()

655
tests/test_train_mono3d.py Executable file
View File

@@ -0,0 +1,655 @@
import json
from pathlib import Path
import cv2
import numpy as np
import pytest
from PIL import Image
from train_mono3d import resolve_data_yaml_for_roi
from ultralytics.data.dataset import Ground3DCalibrationError, YOLOGround3DDataset
from ultralytics.data.ground3d_augment import read_calib_from_path
from ultralytics.utils import YAML
def write_dataset_yaml(path: Path) -> None:
YAML.save(
file=path,
data={
"path": "/tmp/dataset",
"train": "train.txt",
"val": "val.txt",
"class_map": {"car": 0},
"default_roi": "roi0",
"roi_configs": {
"roi0": {
"roi": [1920, 880],
"virtual_fx": 537,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
"roi1": {
"roi": [768, 352],
"virtual_fx": 537,
"virtual_camera_prob": 0.5,
"virtual_camera_val_zoom": True,
"crop_center_mode": "vxvy",
},
},
},
)
def write_clip_level_camera4(calib_dir: Path, image_size: tuple[int, int], focal_u: float = 50.0) -> Path:
camera4_file = calib_dir / "L2_calib" / "camera4.json"
camera4_file.parent.mkdir(parents=True, exist_ok=True)
camera4_file.write_text(
json.dumps(
{
"focal_u": focal_u,
"focal_v": focal_u,
"cu": image_size[0] / 2,
"cv": image_size[1] / 2,
"pitch": 0.0,
"distort_coeffs": [],
}
),
encoding="utf-8",
)
return camera4_file
def create_ground3d_dataset(
tmp_path: Path,
image_sizes: list[tuple[int, int]],
imgsz: tuple[int, int] = (64, 32),
roi: tuple[int, int] | None = None,
ori_img_size: tuple[int, int] | None = None,
) -> tuple[YOLOGround3DDataset, list[str]]:
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_labels = []
image_files = []
roi = roi or imgsz
ori_img_size = ori_img_size or imgsz
for idx, image_size in enumerate(image_sizes, start=1):
rel_label = Path(f"labels/seq{idx}/frame_{idx:04d}.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / f"seq{idx}" / f"frame_{idx:04d}.png"
clip_calib_dir = gt_root / "calib" / f"seq{idx}"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text("car 0.5 0.5 0.25 0.25 0\n", encoding="utf-8")
Image.new("RGB", image_size, color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(clip_calib_dir, image_size)
rel_labels.append(rel_label.as_posix())
image_files.append(str(image_file.resolve()))
(gt_root / "train.txt").write_text("\n".join(rel_labels) + "\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=list(imgsz),
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"roi": list(roi),
"ori_img_size": list(ori_img_size),
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
return dataset, image_files
def test_resolve_data_yaml_for_roi_uses_default_roi(tmp_path):
data_yaml = tmp_path / "mono3d_ground.yaml"
write_dataset_yaml(data_yaml)
resolved_path, selected_roi = resolve_data_yaml_for_roi(str(data_yaml), None)
assert selected_roi == "roi0"
assert resolved_path != str(data_yaml)
resolved_cfg = YAML.load(resolved_path)
assert resolved_cfg["roi"] == [1920, 880]
assert resolved_cfg["virtual_camera_prob"] == -1.0
assert resolved_cfg["crop_center_mode"] == "cxvy"
assert "default_roi" not in resolved_cfg
assert "roi_configs" not in resolved_cfg
def test_resolve_data_yaml_for_roi_supports_explicit_override(tmp_path):
data_yaml = tmp_path / "mono3d_ground.yaml"
write_dataset_yaml(data_yaml)
resolved_path, selected_roi = resolve_data_yaml_for_roi(str(data_yaml), "roi1")
assert selected_roi == "roi1"
resolved_cfg = YAML.load(resolved_path)
assert resolved_cfg["roi"] == [768, 352]
assert resolved_cfg["virtual_camera_prob"] == 0.5
assert resolved_cfg["virtual_camera_val_zoom"] is True
assert resolved_cfg["crop_center_mode"] == "vxvy"
def test_resolve_data_yaml_for_roi_uses_unique_temp_paths(tmp_path):
data_yaml = tmp_path / "mono3d_ground.yaml"
write_dataset_yaml(data_yaml)
resolved_path_a, selected_roi_a = resolve_data_yaml_for_roi(str(data_yaml), "roi1")
resolved_path_b, selected_roi_b = resolve_data_yaml_for_roi(str(data_yaml), "roi1")
assert selected_roi_a == "roi1"
assert selected_roi_b == "roi1"
assert resolved_path_a != resolved_path_b
def test_resolve_data_yaml_for_roi_rejects_unknown_preset(tmp_path):
data_yaml = tmp_path / "mono3d_ground.yaml"
write_dataset_yaml(data_yaml)
with pytest.raises(ValueError, match="Available presets: roi0, roi1"):
resolve_data_yaml_for_roi(str(data_yaml), "roi2")
def test_resolve_data_yaml_for_roi_rejects_missing_required_ground3d_fields(tmp_path):
data_yaml = tmp_path / "mono3d_ground.yaml"
write_dataset_yaml(data_yaml)
data_cfg = YAML.load(data_yaml)
del data_cfg["roi_configs"]["roi1"]["crop_center_mode"]
YAML.save(data_yaml, data_cfg)
with pytest.raises(ValueError, match="crop_center_mode"):
resolve_data_yaml_for_roi(str(data_yaml), "roi1")
def test_ground3d_dataset_resolves_gt_list_to_image_and_calib(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.png"
clip_calib_dir = gt_root / "calib" / "seq0"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text("car 0.5 0.5 0.25 0.25 0\n", encoding="utf-8")
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(clip_calib_dir, (64, 32))
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
assert len(dataset.labels) == 1
assert dataset.labels[0] == (str(gt_root.resolve()), rel_label.as_posix())
raw_calib = read_calib_from_path(
str(image_file.resolve()),
image_root=image_root,
extra_calib_candidates=[str((gt_root / "calib" / "seq0" / "frame_0001.json").resolve())],
)
assert raw_calib["focal_u"] == 50.0
sample = dataset.get_image_and_label(0)
assert sample["im_file"] == str(image_file.resolve())
assert sample["img"].shape[:2] == (32, 64)
assert sample["calib"]["fx"] == pytest.approx(50.0)
def test_ground3d_dataset_prefers_label_root_calibration_over_image_root(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.png"
label_calib_dir = gt_root / "calib" / "seq0"
image_calib_file = image_root / "calib" / "seq0" / "frame_0001.json"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
image_calib_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text("car 0.5 0.5 0.25 0.25 0\n", encoding="utf-8")
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(label_calib_dir, (64, 32), focal_u=80.0)
image_calib_file.write_text(
json.dumps({"focal_u": 50.0, "focal_v": 50.0, "cu": 32.0, "cv": 16.0, "pitch": 0.0, "distort_coeffs": []}),
encoding="utf-8",
)
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
sample = dataset.get_image_and_label(0)
assert sample["calib"]["fx"] == pytest.approx(80.0)
def test_ground3d_dataset_reads_clip_level_camera4_from_label_root(tmp_path):
gt_root = tmp_path / "gt_20260320"
image_root = tmp_path / "dataset_20260202"
rel_label = Path("seq0/clip0/labels/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "seq0" / "clip0" / "images" / "frame_0001.png"
clip_calib_file = gt_root / "seq0" / "clip0" / "calib" / "L2_calib" / "camera4.json"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
clip_calib_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text(
"car 0.5 0.5 0.25 0.25 1 2 3 4 5 6 0.1 0.2 0.3 9 10 11 12 0\n",
encoding="utf-8",
)
Image.new("RGB", (1920, 1080), color=(32, 64, 96)).save(image_file)
clip_calib_file.write_text(
json.dumps(
{
"focal_u": 1450.9230324555967,
"focal_v": 1458.0023697476843,
"cu": 949.5149041625389,
"cv": 569.9146363123367,
"distort_coeffs": [-0.6, 0.7, -0.5, 0.2],
"pitch": 0.214,
"roll": 1.077,
"yaw": -0.643,
}
),
encoding="utf-8",
)
(gt_root / "train.txt").write_text(f"./{rel_label.as_posix()}\n", encoding="utf-8")
raw_calib = read_calib_from_path(str(image_file.resolve()), image_root=image_root, extra_calib_candidates=[
str((gt_root / "seq0" / "clip0" / "calib" / "frame_0001.json").resolve())
])
assert raw_calib is not None
assert raw_calib["focal_u"] == pytest.approx(1450.9230324555967)
assert raw_calib["pitch"] == pytest.approx(np.deg2rad(0.214))
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[768, 352],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"complete_3d_classes": [0],
"roi": [768, 352],
"ori_img_size": [1920, 1080],
"virtual_fx": 537,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
sample = dataset.get_image_and_label(0)
assert sample["im_file"] == str(image_file.resolve())
assert sample["calib"]["fx"] > 0
def test_ground3d_dataset_applies_class_filter_and_rect_lazily(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.png"
clip_calib_dir = gt_root / "calib" / "seq0"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text(
"car 0.5 0.5 0.25 0.25 0\ntruck 0.4 0.4 0.2 0.2 0\n",
encoding="utf-8",
)
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(clip_calib_dir, (64, 32))
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=True,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
classes=[1],
data={
"path": str(image_root),
"class_map": {"car": 0, "truck": 1},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
assert dataset.batch.shape == (1,)
assert dataset.batch_shapes.shape == (1, 2)
sample = dataset.get_image_and_label(0)
assert sample["cls"].reshape(-1).tolist() == [1.0]
def test_ground3d_dataset_keeps_missing_3d_targets_as_nan_for_2d_only_labels(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.png"
clip_calib_dir = gt_root / "calib" / "seq0"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text("car 0.5 0.5 0.25 0.25 1 0\n", encoding="utf-8")
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(clip_calib_dir, (64, 32))
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
raw_sample = dataset.get_image_and_label(0)
assert raw_sample["labels_3d"].shape == (1, 42)
assert np.isnan(raw_sample["labels_3d"]).all()
sample = dataset[0]
assert sample["labels_3d"].shape == (1, 42)
assert sample["labels_3d"].isnan().all().item()
def test_ground3d_dataset_falls_back_to_jpg_when_png_is_missing(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.jpg"
clip_calib_dir = gt_root / "calib" / "seq0"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
label_file.write_text("car 0.5 0.5 0.25 0.25 0\n", encoding="utf-8")
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
write_clip_level_camera4(clip_calib_dir, (64, 32))
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
sample = dataset.get_image_and_label(0)
assert sample["im_file"] == str(image_file.resolve())
assert sample["img"].shape[:2] == (32, 64)
def test_ground3d_dataset_skips_to_next_image_when_imread_fails(tmp_path, monkeypatch):
dataset, image_files = create_ground3d_dataset(tmp_path, [(64, 32), (64, 32)])
original_imread = cv2.imread
def fake_imread(path, flags):
if str(path) == image_files[0]:
return None
return original_imread(path, flags)
monkeypatch.setattr(cv2, "imread", fake_imread)
sample = dataset[0]
assert sample["im_file"] == image_files[1]
assert dataset._bad_image_mask[0]
def test_ground3d_dataset_allows_missing_calibration_for_2d_only_samples(tmp_path):
dataset, image_files = create_ground3d_dataset(
tmp_path,
[(128, 64), (128, 64)],
imgsz=(64, 32),
roi=(64, 32),
ori_img_size=(128, 64),
)
first_calib = tmp_path / "gt" / "calib" / "seq1" / "L2_calib" / "camera4.json"
first_calib.unlink()
sample = dataset.get_image_and_label(0)
assert sample["im_file"] == image_files[0]
assert sample["ori_shape"] == (32, 64)
assert sample["img"].shape[:2] == (32, 64)
assert sample["camera_mode"] == "roi"
def test_ground3d_dataset_fails_on_missing_calibration_for_3d_samples(tmp_path):
gt_root = tmp_path / "gt"
image_root = tmp_path / "dataset"
rel_label = Path("labels/seq0/frame_0001.txt")
label_file = gt_root / rel_label
image_file = image_root / "images" / "seq0" / "frame_0001.png"
clip_calib_file = gt_root / "calib" / "seq0" / "L2_calib" / "camera4.json"
label_file.parent.mkdir(parents=True, exist_ok=True)
image_file.parent.mkdir(parents=True, exist_ok=True)
clip_calib_file.parent.mkdir(parents=True, exist_ok=True)
# 19-col complete_3d label: class + 18 numeric fields.
label_file.write_text("car 0.5 0.5 0.25 0.25 1 2 3 4 5 6 0.1 0.2 0.3 9 10 11 12 0\n", encoding="utf-8")
Image.new("RGB", (64, 32), color=(32, 64, 96)).save(image_file)
clip_calib_file.write_text(
json.dumps({"focal_u": 50.0, "focal_v": 50.0, "cu": 32.0, "cv": 16.0, "pitch": 0.0, "distort_coeffs": []}),
encoding="utf-8",
)
(gt_root / "train.txt").write_text(f"{rel_label.as_posix()}\n", encoding="utf-8")
clip_calib_file.unlink()
dataset = YOLOGround3DDataset(
img_path=str(gt_root / "train.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(image_root),
"class_map": {"car": 0},
"complete_3d_classes": [0],
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
"virtual_camera_prob": -1.0,
"crop_center_mode": "cxvy",
},
)
with pytest.raises(Ground3DCalibrationError, match="calibration file not found"):
dataset[0]
def test_ground3d_dataset_rejects_missing_required_ground3d_fields(tmp_path):
with pytest.raises(ValueError, match="virtual_camera_prob, crop_center_mode"):
YOLOGround3DDataset(
img_path=str(tmp_path / "unused.txt"),
imgsz=[64, 32],
batch_size=1,
augment=False,
rect=False,
stride=32,
pad=0.5,
prefix="test: ",
task="detect",
data={
"path": str(tmp_path / "dataset"),
"class_map": {"car": 0},
"roi": [64, 32],
"ori_img_size": [64, 32],
"virtual_fx": 50,
},
)
def test_ground3d_dataset_skips_images_with_invalid_decoded_shape(tmp_path):
dataset, image_files = create_ground3d_dataset(tmp_path, [(32, 16), (64, 32)])
sample = dataset[0]
assert sample["im_file"] == image_files[1]
assert dataset._bad_image_mask[0]
def test_ground3d_dataset_resizes_in_half_steps_for_quarter_scale(tmp_path, monkeypatch):
dataset, _ = create_ground3d_dataset(
tmp_path,
[(256, 128)],
imgsz=(64, 32),
roi=(256, 128),
ori_img_size=(256, 128),
)
original_resize = cv2.resize
resize_calls = []
def tracked_resize(img, dsize, *args, **kwargs):
resize_calls.append(dsize)
return original_resize(img, dsize, *args, **kwargs)
monkeypatch.setattr(cv2, "resize", tracked_resize)
sample = dataset.get_image_and_label(0)
assert resize_calls == [(128, 64), (64, 32)]
assert sample["img"].shape[:2] == (32, 64)
assert sample["calib"]["fx"] == pytest.approx(12.5)
def test_ground3d_dataset_resizes_in_half_steps_then_remainder(tmp_path, monkeypatch):
dataset, _ = create_ground3d_dataset(
tmp_path,
[(160, 80)],
imgsz=(64, 32),
roi=(160, 80),
ori_img_size=(160, 80),
)
original_resize = cv2.resize
resize_calls = []
def tracked_resize(img, dsize, *args, **kwargs):
resize_calls.append(dsize)
return original_resize(img, dsize, *args, **kwargs)
monkeypatch.setattr(cv2, "resize", tracked_resize)
sample = dataset.get_image_and_label(0)
assert resize_calls == [(80, 40), (64, 32)]
assert sample["img"].shape[:2] == (32, 64)
assert sample["calib"]["fx"] == pytest.approx(20.0)

1528
tests/test_two_roi_inference.py Executable file

File diff suppressed because it is too large Load Diff