feat: HSAP platform v2 — modular navigation, quality review, audit log, world model simulation
Major changes: - New frontend (platform/web/): Vite + React 18 + TypeScript + Tailwind - 4-module navigation: 数据送标 / 模型管理 / 车队管理 / 系统管理 - Data catalog with charts (DMS/ADAS/Lane 3-tab view) - Quality review workflow (标注质检): Good/Fine/Bad scoring with auto-advance - Audit enhancements: batch operations, rejection categories, Feishu notifications - Operation audit log (操作日志) - World model simulation studio (仿真工坊) - Dataset version management with snapshots and diff - ADAS 7-class dataset integration (138K images organized + compressed) - User management with Feishu integration and pagination - CRUD/search/filter on all pages, card layout redesign - PIL-optimized image overlay rendering - Auto-snapshot on build, in_review workflow stage - Removed embedded algorithm code (now in workspace)
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
cmake_minimum_required(VERSION 3.12)
|
||||
project(mnn_yolo_cpp)
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
||||
|
||||
include_directories(${CMAKE_CURRENT_LIST_DIR}/include/)
|
||||
|
||||
link_directories(${CMAKE_CURRENT_LIST_DIR}/libs)
|
||||
|
||||
add_executable("main" "${CMAKE_CURRENT_LIST_DIR}/main.cpp")
|
||||
add_executable("main_interpreter" "${CMAKE_CURRENT_LIST_DIR}/main_interpreter.cpp")
|
||||
|
||||
target_link_libraries("main" MNN MNN_Express MNNOpenCV)
|
||||
target_link_libraries("main_interpreter" MNN MNN_Express MNNOpenCV)
|
||||
@@ -0,0 +1,185 @@
|
||||
# YOLOv8 MNN Inference in C++
|
||||
|
||||
Welcome to the Ultralytics YOLOv8 MNN Inference example in C++! This guide will help you get started with leveraging the powerful [Ultralytics YOLOv8](https://docs.ultralytics.com/models/yolov8/) models using the [Alibaba MNN](https://mnn-docs.readthedocs.io/en/latest/) inference engine in your C++ projects. Whether you're looking to enhance performance on CPU hardware or add flexibility to your applications, this example provides a solid foundation. Learn more about optimizing models and deployment strategies on the [Ultralytics blog](https://www.ultralytics.com/blog).
|
||||
|
||||
## 🌟 Features
|
||||
|
||||
- 🚀 **Model Format Support**: Native support for the MNN format.
|
||||
- ⚡ **Precision Options**: Run models in **FP32**, **FP16** ([half-precision](https://www.ultralytics.com/glossary/half-precision)), and **INT8** ([model quantization](https://www.ultralytics.com/glossary/model-quantization)) precisions for optimized performance and reduced resource consumption.
|
||||
- 🔄 **Dynamic Shape Loading**: Easily handle models with dynamic input shapes, a common requirement in many [computer vision](https://www.ultralytics.com/glossary/computer-vision-cv) tasks.
|
||||
- 📦 **Flexible API Usage**: Choose between MNN's high-level [Express API](https://github.com/alibaba/MNN) for a user-friendly interface or the lower-level [Interpreter API](https://mnn-docs.readthedocs.io/en/latest/cpp/Interpreter.html) for fine-grained control.
|
||||
|
||||
## 📋 Dependencies
|
||||
|
||||
To ensure smooth execution, please make sure you have the following dependencies installed:
|
||||
|
||||
| Dependency | Version | Description |
|
||||
| :------------------------------------------------ | :------- | :------------------------------------------------------------------------------- |
|
||||
| [MNN](https://mnn-docs.readthedocs.io/en/latest/) | >=2.0.0 | The core inference engine from Alibaba. |
|
||||
| [C++](https://en.cppreference.com/w/) | >=14 | A modern C++ compiler supporting C++14 features. |
|
||||
| [CMake](https://cmake.org/documentation/) | >=3.12.0 | Cross-platform build system generator required for building MNN and the example. |
|
||||
| [OpenCV](https://opencv.org/) | Optional | Used for image loading and preprocessing within the example (built with MNN). |
|
||||
|
||||
## ⚙️ Build Instructions
|
||||
|
||||
Follow these steps to build the project:
|
||||
|
||||
1. Clone the Ultralytics repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ultralytics/ultralytics.git
|
||||
cd ultralytics/examples/YOLOv8-MNN-CPP
|
||||
```
|
||||
|
||||
2. Clone the [Alibaba MNN repository](https://github.com/alibaba/MNN):
|
||||
|
||||
```bash
|
||||
git clone https://github.com/alibaba/MNN.git
|
||||
cd MNN
|
||||
```
|
||||
|
||||
3. Build the MNN library:
|
||||
|
||||
```bash
|
||||
# Create build directory
|
||||
mkdir build && cd build
|
||||
|
||||
# Configure CMake (enable OpenCV integration, disable shared libs, enable image codecs)
|
||||
cmake -DMNN_BUILD_OPENCV=ON -DBUILD_SHARED_LIBS=OFF -DMNN_IMGCODECS=ON ..
|
||||
|
||||
# Build the library (use -j flag for parallel compilation)
|
||||
make -j$(nproc) # Use nproc for Linux, sysctl -n hw.ncpu for macOS
|
||||
```
|
||||
|
||||
**Note:** If you encounter issues during the build process, consult the official [MNN documentation](https://mnn-docs.readthedocs.io/en/latest/) for detailed build instructions and troubleshooting tips.
|
||||
|
||||
4. Copy the required MNN libraries and headers to the example project directory:
|
||||
|
||||
```bash
|
||||
# Navigate back to the example directory
|
||||
cd ../..
|
||||
|
||||
# Create directories for libraries and headers if they don't exist
|
||||
mkdir -p libs include
|
||||
|
||||
# Copy static libraries
|
||||
cp MNN/build/libMNN.a libs/ # Main MNN library
|
||||
cp MNN/build/express/libMNN_Express.a libs/ # MNN Express API library
|
||||
cp MNN/build/tools/cv/libMNNOpenCV.a libs/ # MNN OpenCV wrapper library
|
||||
|
||||
# Copy header files
|
||||
cp -r MNN/include .
|
||||
cp -r MNN/tools/cv/include . # MNN OpenCV wrapper headers
|
||||
```
|
||||
|
||||
**Note:**
|
||||
- The library file extensions (`.a` for static) and paths might vary based on your operating system (e.g., use `.lib` on Windows) and build configuration. Adjust the commands accordingly.
|
||||
- This example uses static linking (`.a` files). If you built shared libraries (`.so`, `.dylib`, `.dll`), ensure they are correctly placed or accessible in your system's library path.
|
||||
|
||||
5. Create a build directory for the example project and compile using CMake:
|
||||
```bash
|
||||
mkdir build && cd build
|
||||
cmake ..
|
||||
make
|
||||
```
|
||||
|
||||
## 🔄 Exporting YOLOv8 Models
|
||||
|
||||
To use your Ultralytics YOLOv8 model with this C++ example, you first need to export it to the MNN format. This can be done easily using the `yolo export` command provided by the Ultralytics Python package.
|
||||
|
||||
Refer to the [Ultralytics Export documentation](https://docs.ultralytics.com/modes/export/) for detailed instructions and options.
|
||||
|
||||
```bash
|
||||
# Export a YOLOv8n model to MNN format with input size 640x640
|
||||
yolo export model=yolov8n.pt imgsz=640 format=mnn
|
||||
```
|
||||
|
||||
Alternatively, you can use the `MNNConvert` tool provided by MNN:
|
||||
|
||||
```bash
|
||||
# Assuming MNNConvert is built and in your PATH or MNN build directory
|
||||
# Convert an ONNX model (first export YOLOv8 to ONNX)
|
||||
yolo export model=yolov8n.pt format=onnx
|
||||
./MNN/build/MNNConvert -f ONNX --modelFile yolov8n.onnx --MNNModel yolov8n.mnn --bizCode biz
|
||||
```
|
||||
|
||||
For more details on model conversion using MNN tools, see the [MNN Convert documentation](https://mnn-docs.readthedocs.io/en/latest/tools/convert.html).
|
||||
|
||||
## 🛠️ Usage
|
||||
|
||||
### Ultralytics CLI in Python (for comparison)
|
||||
|
||||
You can verify the exported MNN model using the Ultralytics Python package for a quick check.
|
||||
|
||||
Download an example image:
|
||||
|
||||
```bash
|
||||
wget https://ultralytics.com/images/bus.jpg
|
||||
```
|
||||
|
||||
Run prediction using the MNN model:
|
||||
|
||||
```bash
|
||||
yolo predict model=yolov8n.mnn source=bus.jpg
|
||||
```
|
||||
|
||||
Expected Python Output:
|
||||
|
||||
```
|
||||
ultralytics/examples/YOLOv8-MNN-CPP/assets/bus.jpg: 640x640 4 persons, 1 bus, 84.6ms
|
||||
Speed: 9.7ms preprocess, 128.7ms inference, 12.4ms postprocess per image at shape (1, 3, 640, 640)
|
||||
Results saved to runs/detect/predict
|
||||
```
|
||||
|
||||
_(Note: Speed and specific detections might vary based on hardware and model version)_
|
||||
|
||||
### MNN Express API in C++
|
||||
|
||||
This example uses the higher-level Express API for simpler inference code.
|
||||
|
||||
```bash
|
||||
./build/main yolov8n.mnn bus.jpg
|
||||
```
|
||||
|
||||
Expected C++ Express API Output:
|
||||
|
||||
```
|
||||
The device supports: i8sdot:0, fp16:0, i8mm: 0, sve2: 0, sme2: 0
|
||||
Detection: box = {48.63, 399.30, 243.65, 902.90}, class = person, score = 0.86
|
||||
Detection: box = {22.14, 228.36, 796.07, 749.74}, class = bus, score = 0.86
|
||||
Detection: box = {669.92, 375.82, 809.86, 874.41}, class = person, score = 0.86
|
||||
Detection: box = {216.01, 405.24, 346.36, 858.19}, class = person, score = 0.82
|
||||
Detection: box = {-0.11, 549.41, 62.05, 874.88}, class = person, score = 0.33
|
||||
Result image write to `mnn_yolov8_cpp.jpg`.
|
||||
Speed: 35.6ms preprocess, 386.0ms inference, 68.3ms postprocess
|
||||
```
|
||||
|
||||
_(Note: Speed and specific detections might vary based on hardware and MNN configuration)_
|
||||
|
||||
### MNN Interpreter API in C++
|
||||
|
||||
This example uses the lower-level Interpreter API, offering more control over the inference process.
|
||||
|
||||
```bash
|
||||
./build/main_interpreter yolov8n.mnn bus.jpg
|
||||
```
|
||||
|
||||
Expected C++ Interpreter API Output:
|
||||
|
||||
```
|
||||
The device supports: i8sdot:0, fp16:0, i8mm: 0, sve2: 0, sme2: 0
|
||||
Detection: box = {48.63, 399.30, 243.65, 902.90}, class = person, score = 0.86
|
||||
Detection: box = {22.14, 228.36, 796.07, 749.74}, class = bus, score = 0.86
|
||||
Detection: box = {669.92, 375.82, 809.86, 874.41}, class = person, score = 0.86
|
||||
Detection: box = {216.01, 405.24, 346.36, 858.19}, class = person, score = 0.82
|
||||
Result image written to `mnn_yolov8_cpp.jpg`.
|
||||
Speed: 26.0ms preprocess, 190.9ms inference, 58.9ms postprocess
|
||||
```
|
||||
|
||||
_(Note: Speed and specific detections might vary based on hardware and MNN configuration)_
|
||||
|
||||
## ❤️ Contributions
|
||||
|
||||
We hope this example helps you integrate Ultralytics YOLOv8 with MNN into your C++ projects effortlessly! Contributions to improve this example or add new features are highly welcome. Please see the [Ultralytics contribution guidelines](https://docs.ultralytics.com/help/contributing/) for more information on how to get involved.
|
||||
|
||||
For further guides, tutorials, and documentation on Ultralytics YOLO models and tools, visit the main [Ultralytics documentation](https://docs.ultralytics.com/). Happy coding! 🚀
|
||||
@@ -0,0 +1,246 @@
|
||||
// Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <MNN/expr/Module.hpp>
|
||||
#include <MNN/expr/Executor.hpp>
|
||||
#include <MNN/expr/ExprCreator.hpp>
|
||||
#include <cv/cv.hpp>
|
||||
|
||||
using namespace MNN;
|
||||
using namespace MNN::Express;
|
||||
using namespace MNN::CV;
|
||||
|
||||
class Inference {
|
||||
public:
|
||||
|
||||
// Load model: Create runtime, set cache if needed, and load the model file.
|
||||
bool loadModel(const std::string &modelPath,
|
||||
int forwardType = MNN_FORWARD_CPU,
|
||||
int precision = 0,
|
||||
int thread = 4) {
|
||||
MNN::ScheduleConfig sConfig;
|
||||
sConfig.type = static_cast<MNNForwardType>(forwardType);
|
||||
sConfig.numThread = thread;
|
||||
BackendConfig bConfig;
|
||||
bConfig.precision = static_cast<BackendConfig::PrecisionMode>(precision);
|
||||
sConfig.backendConfig = &bConfig;
|
||||
|
||||
std::shared_ptr<Executor::RuntimeManager> rtmgr(
|
||||
Executor::RuntimeManager::createRuntimeManager(sConfig)
|
||||
);
|
||||
if (rtmgr == nullptr) {
|
||||
MNN_ERROR("Empty RuntimeManager\n");
|
||||
return false;
|
||||
}
|
||||
rtmgr->setCache(".cachefile");
|
||||
net = std::shared_ptr<Module>(Module::load(std::vector<std::string>{},
|
||||
std::vector<std::string>{}, modelPath.c_str(), rtmgr));
|
||||
if (net == nullptr) {
|
||||
return false;
|
||||
}
|
||||
runtimeManager = rtmgr;
|
||||
const Module::Info* info = net->getInfo();
|
||||
if (info == nullptr) {
|
||||
MNN_ERROR("Empty Module Info\n");
|
||||
return false;
|
||||
}
|
||||
// Parse bizCode to extract class names.
|
||||
if (info->bizCode.empty()) {
|
||||
MNN_ERROR("Empty bizCode\n");
|
||||
classNames.clear();
|
||||
return false;
|
||||
}
|
||||
// Get imgsz from bizCode.
|
||||
auto imgsz_start = info->bizCode.find("\"imgsz\": [");
|
||||
if (imgsz_start == std::string::npos) {
|
||||
MNN_PRINT("No imgsz found in bizCode, setting classNames empty.\n");
|
||||
} else {
|
||||
auto imgsz_end = info->bizCode.find("]", imgsz_start);
|
||||
if (imgsz_end == std::string::npos) {
|
||||
MNN_PRINT("No closing bracket for imgsz in bizCode, setting classNames empty.\n");
|
||||
} else {
|
||||
std::string imgszText = info->bizCode.substr(imgsz_start + 10, imgsz_end - imgsz_start - 10);
|
||||
std::vector<std::string> imgszVec;
|
||||
std::stringstream ss(imgszText);
|
||||
std::string item;
|
||||
while (std::getline(ss, item, ',')) {
|
||||
imgszVec.push_back(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Get names from bizCode.
|
||||
auto names_start = info->bizCode.find("\"names\": {");
|
||||
if (names_start == std::string::npos) {
|
||||
MNN_PRINT("No names found in bizCode, setting classNames empty.\n");
|
||||
classNames.clear();
|
||||
} else {
|
||||
auto names_end = info->bizCode.find("}", names_start);
|
||||
if (names_end == std::string::npos) {
|
||||
MNN_PRINT("No closing brace for names in bizCode, setting classNames empty.\n");
|
||||
classNames.clear();
|
||||
} else {
|
||||
std::string namesDict = info->bizCode.substr(names_start + 10, names_end - names_start - 10);
|
||||
parseClassNamesFromBizCode(namesDict);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void parseImgszFromBizCode(const std::string& bizText) {
|
||||
std::regex rgx("\"imgsz\":\\s*\\[(\\d+),\\s*(\\d+)\\]");
|
||||
std::smatch match;
|
||||
if (std::regex_search(bizText, match, rgx)) {
|
||||
int ih = std::stoi(match[1].str());
|
||||
int iw = std::stoi(match[2].str());
|
||||
MNN_PRINT("Input size: %d x %d\n", iw, ih);
|
||||
} else {
|
||||
MNN_PRINT("No imgsz found in bizCode.\n");
|
||||
}
|
||||
}
|
||||
|
||||
void parseClassNamesFromBizCode(const std::string& bizText) {
|
||||
std::regex rgx("\"(\\d+)\"\\s*:\\s*\"([^\"]+)\"");
|
||||
std::smatch match;
|
||||
std::string s = bizText;
|
||||
classNames.clear();
|
||||
while (std::regex_search(s, match, rgx)) {
|
||||
int index = std::stoi(match[1].str());
|
||||
std::string name = match[2].str();
|
||||
if (classNames.size() <= static_cast<size_t>(index)) {
|
||||
classNames.resize(index + 1);
|
||||
}
|
||||
classNames[index] = name;
|
||||
s = match.suffix().str();
|
||||
}
|
||||
}
|
||||
|
||||
VARP preprocess(VARP &originalImage, float &scale) {
|
||||
auto dims = originalImage->getInfo()->dim;
|
||||
int ih = dims[0];
|
||||
int iw = dims[1];
|
||||
int targetWidth = std::stoi(imgszVec[0]);
|
||||
int targetHeight = std::stoi(imgszVec[1]);
|
||||
int len = ih > iw ? ih : iw;
|
||||
scale = static_cast<float>(len) / std::max(targetWidth, targetHeight);
|
||||
std::vector<int> padvals { 0, len - ih, 0, len - iw, 0, 0 };
|
||||
auto pads = _Const(static_cast<void*>(padvals.data()), {3, 2}, NCHW, halide_type_of<int>());
|
||||
auto image = _Pad(originalImage, pads, CONSTANT);
|
||||
image = resize(image, Size(targetWidth, targetHeight), 0, 0, INTER_LINEAR, -1, {0., 0., 0.}, {1./255., 1./255., 1./255.});
|
||||
auto input = _Unsqueeze(image, {0});
|
||||
input = _Convert(input, NC4HW4);
|
||||
return input;
|
||||
}
|
||||
|
||||
void runInference(VARP input) {
|
||||
std::vector<VARP> outputs = net->onForward({input});
|
||||
mOutput = outputs[0];
|
||||
}
|
||||
|
||||
void postprocess(float scale, VARP originalImage, float iouThreshold = 0.45, float scoreThreshold = 0.25) {
|
||||
auto output = _Convert(mOutput, NCHW);
|
||||
output = _Squeeze(output);
|
||||
// Expected output shape: [84, 8400]
|
||||
auto cx = _Gather(output, _Scalar<int>(0));
|
||||
auto cy = _Gather(output, _Scalar<int>(1));
|
||||
auto w = _Gather(output, _Scalar<int>(2));
|
||||
auto h = _Gather(output, _Scalar<int>(3));
|
||||
|
||||
std::vector<int> startvals { 4, 0 };
|
||||
auto start = _Const(static_cast<void*>(startvals.data()), {2}, NCHW, halide_type_of<int>());
|
||||
std::vector<int> sizevals { -1, -1 };
|
||||
auto size = _Const(static_cast<void*>(sizevals.data()), {2}, NCHW, halide_type_of<int>());
|
||||
auto probs = _Slice(output, start, size);
|
||||
// [cx, cy, w, h] -> [x1, y1, x2, y2]
|
||||
auto x1 = cx - w * _Const(0.5);
|
||||
auto y1 = cy - h * _Const(0.5);
|
||||
auto x2 = cx + w * _Const(0.5);
|
||||
auto y2 = cy + h * _Const(0.5);
|
||||
auto boxes = _Stack({x1, y1, x2, y2}, 1);
|
||||
auto scores = _ReduceMax(probs, {0});
|
||||
auto ids = _ArgMax(probs, 0);
|
||||
auto result_ids = _Nms(boxes, scores, 100, 0.45, 0.25);
|
||||
auto result_ptr = result_ids->readMap<int>();
|
||||
auto box_ptr = boxes->readMap<float>();
|
||||
auto ids_ptr = ids->readMap<int>();
|
||||
auto score_ptr = scores->readMap<float>();
|
||||
for (int i = 0; i < 100; i++) {
|
||||
auto idx = result_ptr[i];
|
||||
if (idx < 0) break;
|
||||
auto x1 = box_ptr[idx * 4 + 0] * scale;
|
||||
auto y1 = box_ptr[idx * 4 + 1] * scale;
|
||||
auto x2 = box_ptr[idx * 4 + 2] * scale;
|
||||
auto y2 = box_ptr[idx * 4 + 3] * scale;
|
||||
auto class_idx = ids_ptr[idx];
|
||||
auto score = score_ptr[idx];
|
||||
printf("Detection: box = {%.2f, %.2f, %.2f, %.2f}, class = %s, score = %.2f\n",
|
||||
x1, y1, x2, y2, classNames[class_idx].c_str(), score);
|
||||
rectangle(originalImage, { x1, y1 }, { x2, y2 }, { 0, 255, 0 }, 2);
|
||||
}
|
||||
if (imwrite("mnn_yolov8_cpp.jpg", originalImage)) {
|
||||
MNN_PRINT("Result image written to `mnn_yolov8_cpp.jpg`.\n");
|
||||
}
|
||||
}
|
||||
|
||||
// Update runtime cache.
|
||||
void updateCache() {
|
||||
if (runtimeManager)
|
||||
runtimeManager->updateCache();
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<Module> net;
|
||||
VARP mOutput;
|
||||
std::shared_ptr<Executor::RuntimeManager> runtimeManager;
|
||||
std::vector<std::string> classNames;
|
||||
std::vector<std::string> imgszVec { "640", "640" };
|
||||
};
|
||||
|
||||
int main(int argc, const char* argv[]) {
|
||||
if (argc < 3) {
|
||||
MNN_PRINT("Usage: ./main yolov8n.mnn bus.jpg [forwardType] [precision] [thread]\n");
|
||||
return 0;
|
||||
}
|
||||
int thread = 4;
|
||||
int precision = 0;
|
||||
int forwardType = MNN_FORWARD_CPU;
|
||||
if (argc >= 4) {
|
||||
forwardType = atoi(argv[3]);
|
||||
}
|
||||
if (argc >= 5) {
|
||||
precision = atoi(argv[4]);
|
||||
}
|
||||
if (argc >= 6) {
|
||||
thread = atoi(argv[5]);
|
||||
}
|
||||
|
||||
Inference infer;
|
||||
if (!infer.loadModel(argv[1], forwardType, precision, thread))
|
||||
return 1;
|
||||
|
||||
const clock_t t0 = clock();
|
||||
float scale = 1.0f;
|
||||
VARP originalImage = imread(argv[2]);
|
||||
VARP input = infer.preprocess(originalImage, scale);
|
||||
double preprocess_time = 1000.0 * (clock() - t0) / CLOCKS_PER_SEC;
|
||||
|
||||
const clock_t t1 = clock();
|
||||
infer.runInference(input);
|
||||
double inference_time = 1000.0 * (clock() - t1) / CLOCKS_PER_SEC;
|
||||
|
||||
const clock_t t2 = clock();
|
||||
infer.postprocess(scale, originalImage);
|
||||
double postprocess_time = 1000.0 * (clock() - t2) / CLOCKS_PER_SEC;
|
||||
|
||||
printf("Speed: %.1fms preprocess, %.1fms inference, %.1fms postprocess\n",
|
||||
preprocess_time, inference_time, postprocess_time);
|
||||
|
||||
infer.updateCache();
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
// Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <regex>
|
||||
#include <MNN/ImageProcess.hpp>
|
||||
#include <MNN/expr/Module.hpp>
|
||||
#include <MNN/expr/Executor.hpp>
|
||||
#include <MNN/expr/ExprCreator.hpp>
|
||||
#include <cv/cv.hpp>
|
||||
|
||||
using namespace MNN;
|
||||
using namespace MNN::Express;
|
||||
using namespace MNN::CV;
|
||||
|
||||
class Inference {
|
||||
public:
|
||||
Inference() : interpreter(nullptr), session(nullptr), inputTensor(nullptr) {
|
||||
inputDims = {1, 3, 640, 640};
|
||||
}
|
||||
|
||||
~Inference() {
|
||||
if(interpreter) {
|
||||
delete interpreter;
|
||||
interpreter = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
// Load model, create session, and resize the input tensor.
|
||||
bool loadModel(const std::string &modelPath,
|
||||
int forwardType = MNN_FORWARD_CPU,
|
||||
int precision = 1,
|
||||
int thread = 4) {
|
||||
MNN::ScheduleConfig sConfig;
|
||||
sConfig.type = static_cast<MNNForwardType>(forwardType);
|
||||
sConfig.numThread = thread;
|
||||
BackendConfig bConfig;
|
||||
bConfig.precision = static_cast<BackendConfig::PrecisionMode>(precision);
|
||||
sConfig.backendConfig = &bConfig;
|
||||
|
||||
interpreter = MNN::Interpreter::createFromFile(modelPath.c_str());
|
||||
if (!interpreter) {
|
||||
MNN_PRINT("Error: Failed to create interpreter from model file.\n");
|
||||
return false;
|
||||
}
|
||||
session = interpreter->createSession(sConfig);
|
||||
if(!session) {
|
||||
MNN_PRINT("Error: Failed to create session.\n");
|
||||
return false;
|
||||
}
|
||||
inputTensor = interpreter->getSessionInput(session, "images");
|
||||
interpreter->resizeTensor(inputTensor, inputDims);
|
||||
interpreter->resizeSession(session);
|
||||
std::string bizCode = interpreter->bizCode();
|
||||
|
||||
// Get names from bizCode.
|
||||
auto names_start = bizCode.find("\"names\": {");
|
||||
if (names_start == std::string::npos) {
|
||||
MNN_PRINT("No names found in bizCode, setting classNames empty.\n");
|
||||
classNames.clear();
|
||||
} else {
|
||||
auto names_end = bizCode.find("}", names_start);
|
||||
if (names_end == std::string::npos) {
|
||||
MNN_PRINT("No closing brace for names in bizCode, setting classNames empty.\n");
|
||||
classNames.clear();
|
||||
} else {
|
||||
std::string namesDict = bizCode.substr(names_start + 10, names_end - names_start - 10);
|
||||
parseClassNamesFromBizCode(namesDict);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void parseClassNamesFromBizCode(const std::string& bizText) {
|
||||
std::regex rgx("\"(\\d+)\"\\s*:\\s*\"([^\"]+)\"");
|
||||
std::smatch match;
|
||||
std::string s = bizText;
|
||||
classNames.clear();
|
||||
while (std::regex_search(s, match, rgx)) {
|
||||
int index = std::stoi(match[1].str());
|
||||
std::string name = match[2].str();
|
||||
if (classNames.size() <= static_cast<size_t>(index)) {
|
||||
classNames.resize(index + 1);
|
||||
}
|
||||
classNames[index] = name;
|
||||
s = match.suffix().str();
|
||||
}
|
||||
}
|
||||
|
||||
VARP preprocess(VARP &originalImage, int targetSize, float &scale) {
|
||||
const auto dims = originalImage->getInfo()->dim;
|
||||
const int ih = dims[0], iw = dims[1];
|
||||
const int len = (ih >= iw ? ih : iw);
|
||||
scale = static_cast<float>(len) / targetSize;
|
||||
|
||||
// Use fixed-size array for padding values.
|
||||
int padvals[6] = { 0, len - ih, 0, len - iw, 0, 0 };
|
||||
auto pads = _Const(static_cast<void*>(padvals), {3, 2}, NCHW, halide_type_of<int>());
|
||||
auto padded = _Pad(originalImage, pads, CONSTANT);
|
||||
|
||||
auto resized = MNN::CV::resize(padded, MNN::CV::Size(targetSize, targetSize),
|
||||
0, 0, MNN::CV::INTER_LINEAR, -1,
|
||||
{0.f, 0.f, 0.f},
|
||||
{1.f/255, 1.f/255, 1.f/255});
|
||||
|
||||
// Chain unsqueeze and conversion
|
||||
auto input = _Unsqueeze(resized, {0});
|
||||
input = _Convert(input, NCHW);
|
||||
return input;
|
||||
}
|
||||
|
||||
// Run inference by copying preprocessed data into input tensor.
|
||||
void runInference(VARP input) {
|
||||
auto tmp_input = MNN::Tensor::create(inputDims, halide_type_of<float>(),
|
||||
const_cast<void*>(input->readMap<void>()),
|
||||
MNN::Tensor::CAFFE);
|
||||
inputTensor->copyFromHostTensor(tmp_input);
|
||||
interpreter->runSession(session);
|
||||
}
|
||||
|
||||
// Postprocess the output, perform NMS, and draw bounding boxes on originalImage.
|
||||
void postprocess(float scale, VARP originalImage, float modelScoreThreshold = 0.25, float modelNMSThreshold = 0.45) {
|
||||
auto outputTensor = interpreter->getSessionOutput(session, "output0");
|
||||
|
||||
// ---------------- Post Processing ----------------
|
||||
auto outputs = outputTensor->host<float>();
|
||||
auto outputVar = _Const(outputs, outputTensor->shape(), NCHW, halide_type_of<float>());
|
||||
auto output = _Squeeze(_Convert(outputVar, NCHW));
|
||||
|
||||
// Expected output shape: [84, 8400] where first 4 rows are [cx, cy, w, h].
|
||||
auto cx = _Gather(output, _Scalar<int>(0));
|
||||
auto cy = _Gather(output, _Scalar<int>(1));
|
||||
auto w = _Gather(output, _Scalar<int>(2));
|
||||
auto h = _Gather(output, _Scalar<int>(3));
|
||||
|
||||
// Slice probability values (starting at row 4).
|
||||
const int startArr[2] = { 4, 0 };
|
||||
const int sizeArr[2] = { -1, -1 };
|
||||
auto start = _Const(static_cast<void*>(const_cast<int*>(startArr)), {2}, NCHW, halide_type_of<int>());
|
||||
auto size = _Const(static_cast<void*>(const_cast<int*>(sizeArr)), {2}, NCHW, halide_type_of<int>());
|
||||
auto probs = _Slice(output, start, size);
|
||||
|
||||
// Convert [cx, cy, w, h] to [y1, x1, y2, x2] using half-width/height.
|
||||
auto half = _Const(0.5);
|
||||
auto x1 = cx - w * half;
|
||||
auto y1 = cy - h * half;
|
||||
auto x2 = cx + w * half;
|
||||
auto y2 = cy + h * half;
|
||||
auto boxes = _Stack({x1, y1, x2, y2}, 1);
|
||||
|
||||
auto scores = _ReduceMax(probs, {0});
|
||||
auto ids = _ArgMax(probs, 0);
|
||||
auto result_ids = _Nms(boxes, scores, 100, modelScoreThreshold, modelNMSThreshold);
|
||||
|
||||
auto result_ptr = result_ids->readMap<int>();
|
||||
auto box_ptr = boxes->readMap<float>();
|
||||
auto ids_ptr = ids->readMap<int>();
|
||||
auto score_ptr = scores->readMap<float>();
|
||||
|
||||
const int numResults = result_ids->getInfo()->size;
|
||||
for (int i = 0; i < numResults; i++) {
|
||||
int idx = result_ptr[i];
|
||||
if (idx < 0) break;
|
||||
float x1 = box_ptr[idx * 4 + 0] * scale;
|
||||
float y1 = box_ptr[idx * 4 + 1] * scale;
|
||||
float x2 = box_ptr[idx * 4 + 2] * scale;
|
||||
float y2 = box_ptr[idx * 4 + 3] * scale;
|
||||
int class_idx = ids_ptr[idx];
|
||||
float score = score_ptr[idx];
|
||||
printf("Detection: box = {%.2f, %.2f, %.2f, %.2f}, class = %s, score = %.2f\n",
|
||||
x1, y1, x2, y2, classNames[class_idx].c_str(), score);
|
||||
MNN::CV::rectangle(originalImage, { x1, y1 }, { x2, y2 }, { 0, 255, 0 }, 2);
|
||||
// Note: MNN::CV does not offer a putText function.
|
||||
// For text annotations, consider converting the image to cv::Mat and using OpenCV.
|
||||
}
|
||||
if (MNN::CV::imwrite("mnn_yolov8_cpp.jpg", originalImage)) {
|
||||
MNN_PRINT("Result image written to `mnn_yolov8_cpp.jpg`.\n");
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
MNN::Interpreter* interpreter;
|
||||
MNN::Session* session;
|
||||
MNN::Tensor* inputTensor;
|
||||
std::vector<int> inputDims;
|
||||
std::vector<std::string> classNames;
|
||||
};
|
||||
|
||||
int main(int argc, const char* argv[]) {
|
||||
if (argc < 3) {
|
||||
MNN_PRINT("Usage: ./main yolov8n.mnn input.jpg [backend] [precision] [thread]\n");
|
||||
return 0;
|
||||
}
|
||||
int backend = MNN_FORWARD_CPU;
|
||||
int precision = 1;
|
||||
int thread = 4;
|
||||
if (argc >= 4) {
|
||||
backend = atoi(argv[3]);
|
||||
}
|
||||
if (argc >= 5) {
|
||||
precision = atoi(argv[4]);
|
||||
}
|
||||
if (argc >= 6) {
|
||||
thread = atoi(argv[5]);
|
||||
}
|
||||
|
||||
Inference infer;
|
||||
if (!infer.loadModel(argv[1], backend, precision, thread))
|
||||
return 1;
|
||||
|
||||
const clock_t begin_time = clock();
|
||||
float scale = 1.0f;
|
||||
VARP originalImage = imread(argv[2]);
|
||||
VARP input = infer.preprocess(originalImage, 640, scale);
|
||||
auto preprocess_time = 1000.0 * (clock() - begin_time) / CLOCKS_PER_SEC;
|
||||
|
||||
const clock_t begin_time2 = clock();
|
||||
infer.runInference(input);
|
||||
auto inference_time = 1000.0 * (clock() - begin_time2) / CLOCKS_PER_SEC;
|
||||
const clock_t begin_time3 = clock();
|
||||
infer.postprocess(scale, originalImage);
|
||||
|
||||
auto postprocess_time = 1000.0 * (clock() - begin_time3) / CLOCKS_PER_SEC;
|
||||
printf("Speed: %.1fms preprocess, %.1fms inference, %.1fms postprocess\n",
|
||||
preprocess_time, inference_time, postprocess_time);
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user