深度學習之用CelebA_Spoof數據集搭建一個活體檢測-訓練好的模型用MNN來推理

一、模型轉換準備

首先確保已完成PyTorch到ONNX的轉換：深度學習之用CelebA_Spoof數據集搭建活體檢測系統：模型驗證與測試。這里有將PyTorch到ONNX格式的模型轉換。

二、ONNX轉MNN

使用MNN轉換工具進行格式轉換：具體的編譯過程可以參考MNN的官方代碼。MNN是一個輕量級的深度神經網絡引擎，支持深度學習的推理與訓練。適用于服務器/個人電腦/手機/嵌入式各類設備。

./MNNConvert -f ONNX --modelFile live_spoof.onnx --MNNModel live_spoof.mnn

三、C++推理工程搭建

工程結構

mnn_inference/
├── CMakeLists.txt
├── include/
│   ├── InferenceInit.h
│   └── LiveSpoofDetector.h
├── src/
│   ├── InferenceInit.cpp
│   ├── LiveSpoofDetector.cpp
│   └── CMakeLists.txt
└── third_party/MNN/

根目錄下的 CMakeLists.txt

cmake_minimum_required(VERSION 3.12)
project(MNNInference)# 設置C++標準
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)# 查找OpenCV
find_package(OpenCV REQUIRED)# 包含第三方庫MNN
set(MNN_DIR ${CMAKE_SOURCE_DIR}/third_party/MNN)
include_directories(${MNN_DIR}/include)# 添加子目錄
add_subdirectory(src)# 主可執行文件
add_executable(mnn_inference_main src/main.cpp
)# 鏈接庫
target_link_libraries(mnn_inference_mainPRIVATEinference_lib${MNN_DIR}/lib/libMNN.so${OpenCV_LIBS}
)# 安裝規則
install(TARGETS mnn_inference_mainRUNTIME DESTINATION bin
)

src目錄下的 CMakeLists.txt

# 添加庫
add_library(inference_lib STATICInferenceInit.cppLiveSpoofDetector.cpp
)# 包含目錄
target_include_directories(inference_libPUBLIC${CMAKE_SOURCE_DIR}/include${MNN_DIR}/include${OpenCV_INCLUDE_DIRS}
)# 編譯選項
target_compile_options(inference_libPRIVATE-Wall-O3
)

核心實現代碼

將MNN讀取導入模型和一些mnn_session進行預處理的公共部分抽取出來，以后可以更換不同的模型，只需要給出特定的預處理。

// InferenceInit.h
#ifndef MNN_CORE_MNN_HANDLER_H
#define MNN_CORE_MNN_HANDLER_H
#include "MNN/Interpreter.hpp"
#include "MNN/MNNDefine.h"
#include "MNN/Tensor.hpp"
#include "MNN/ImageProcess.hpp"#include <iostream>
#include "opencv2/opencv.hpp"
#endif
#include "mylog.h"
#define LITEMNN_DEBUG
namespace mnncore
{class  BasicMNNHandler{protected:std::shared_ptr<MNN::Interpreter> mnn_interpreter;MNN::Session *mnn_session = nullptr;MNN::Tensor *input_tensor = nullptr; // assume single input.MNN::ScheduleConfig schedule_config;std::shared_ptr<MNN::CV::ImageProcess> pretreat; // init at subclassconst char *log_id = nullptr;const char *mnn_path = nullptr;const char *mnn_model_data = nullptr;//int mnn_model_size = 0;protected:const int num_threads; // initialize at runtime.int input_batch;int input_channel;int input_height;int input_width;int dimension_type;int num_outputs = 1;protected:explicit BasicMNNHandler(const std::string &_mnn_path, int _num_threads = 1);int initialize_handler();std::string turnHeadDataToString(std::string headData);virtual ~BasicMNNHandler();// un-copyableprotected:BasicMNNHandler(const BasicMNNHandler &) = delete; //BasicMNNHandler(BasicMNNHandler &&) = delete; //BasicMNNHandler &operator=(const BasicMNNHandler &) = delete; //BasicMNNHandler &operator=(BasicMNNHandler &&) = delete; //protected:virtual void transform(const cv::Mat &mat) = 0; // ? needed ?private:void print_debug_string();};
}
#endif //MNN_CORE_MNN_HANDLER_H

// InferenceInit.cpp
#include "mnn/core/InferenceInit.h"
namespace mnncore
{
BasicMNNHandler::BasicMNNHandler(const std::string &_mnn_path, int _num_threads) :log_id(_mnn_path.data()), mnn_path(_mnn_path.data()),num_threads(_num_threads)
{//initialize_handler();
}int BasicMNNHandler::initialize_handler()
{std::cout<<"load  Model from file: " << mnn_path << "\n";mnn_interpreter = std::shared_ptr<MNN::Interpreter>(MNN::Interpreter::createFromFile(mnn_path));myLog(ERROR_, "mnn_interpreter createFromFile done!");if (nullptr == mnn_interpreter) {std::cout << "load centerface failed." << std::endl;return -1;}// 2. init schedule_configschedule_config.numThread = (int) num_threads;MNN::BackendConfig backend_config;backend_config.precision = MNN::BackendConfig::Precision_Low; // default Precision_Highbackend_config.memory = MNN::BackendConfig::Memory_Low;backend_config.power = MNN::BackendConfig::Power_Low;schedule_config.backendConfig = &backend_config;// 3. create sessionmyLog(ERROR_, "createSession...");mnn_session = mnn_interpreter->createSession(schedule_config);// 4. init input tensormyLog(ERROR_, "getSessionInput...");input_tensor = mnn_interpreter->getSessionInput(mnn_session, nullptr);// 5. init input dimsinput_batch = input_tensor->batch();input_channel = input_tensor->channel();input_height = input_tensor->height();input_width = input_tensor->width();dimension_type = input_tensor->getDimensionType();myLog(ERROR_, "input_batch: %d, input_channel: %d, input_height: %d, input_width: %d, dimension_type: %d", input_batch, input_channel, input_height, input_width, dimension_type);// 6. resize tensor & session needed ???if (dimension_type == MNN::Tensor::CAFFE){// NCHWmnn_interpreter->resizeTensor(input_tensor, {input_batch, input_channel, input_height, input_width});mnn_interpreter->resizeSession(mnn_session);} // NHWCelse if (dimension_type == MNN::Tensor::TENSORFLOW){mnn_interpreter->resizeTensor(input_tensor, {input_batch, input_height, input_width, input_channel});mnn_interpreter->resizeSession(mnn_session);} // NC4HW4else if (dimension_type == MNN::Tensor::CAFFE_C4){
#ifdef LITEMNN_DEBUGstd::cout << "Dimension Type is CAFFE_C4, skip resizeTensor & resizeSession!\n";
#endif}// output countnum_outputs = (int)mnn_interpreter->getSessionOutputAll(mnn_session).size();
#ifdef LITEMNN_DEBUGthis->print_debug_string();
#endifreturn 0;
}BasicMNNHandler::~BasicMNNHandler()
{mnn_interpreter->releaseModel();if (mnn_session)mnn_interpreter->releaseSession(mnn_session);
}
void BasicMNNHandler::print_debug_string()
{std::cout << "LITEMNN_DEBUG LogId: " << log_id << "\n";std::cout << "=============== Input-Dims ==============\n";if (input_tensor) input_tensor->printShape();if (dimension_type == MNN::Tensor::CAFFE)std::cout << "Dimension Type: (CAFFE/PyTorch/ONNX)NCHW" << "\n";else if (dimension_type == MNN::Tensor::TENSORFLOW)std::cout << "Dimension Type: (TENSORFLOW)NHWC" << "\n";else if (dimension_type == MNN::Tensor::CAFFE_C4)std::cout << "Dimension Type: (CAFFE_C4)NC4HW4" << "\n";std::cout << "=============== Output-Dims ==============\n";auto tmp_output_map = mnn_interpreter->getSessionOutputAll(mnn_session);std::cout << "getSessionOutputAll done!\n";for (auto it = tmp_output_map.cbegin(); it != tmp_output_map.cend(); ++it){std::cout << "Output: " << it->first << ": ";it->second->printShape();}std::cout << "========================================\n";
}
} // namespace mnncore

主要的推理處理代碼：
頭文件聲明

//LiveSpoofDetector.h
#include "mnn/core/InferenceInit.h"
#include <iostream>
#include <fstream>
#include <sstream>
#include<iterator>
#include <algorithm>#define RESIZE_LIVE_SPOOF_SIZE 112
using namespace mnncore;
namespace mnncv {class SqueezeNetLiveSpoof : public BasicMNNHandler{public:explicit SqueezeNetLiveSpoof(const std::string &model_path, int numThread = 1);~SqueezeNetLiveSpoof() override = default;// 保留原有函數int Init(const char* model_path);float detect_handler(const unsigned char* pData, int width, int height, int nchannel,  int mod);cv::Mat m_image;private:void initialize_pretreat();void transform(const cv::Mat &mat) override;std::vector<cv::Point2f> coord5points;const float meanVals_[3] = { 103.94f, 116.78f, 123.68f};const float normVals_[3] = {0.017f, 0.017f, 0.017f};};
}

函數定義：

// LiveSpoofDetector.cpp
#include "include/mnn/cv/RGB/LiveSpoofDetector.h"
#include <opencv2/opencv.hpp>using namespace mnncv;SqueezeNetLiveSpoof::SqueezeNetLiveSpoof(const std::string &model_path, int numThread) : BasicMNNHandler(model_path, numThread) {initialize_pretreat();
}int SqueezeNetLiveSpoof::Init(const char* model_path) {std::string model_path_str = model_path;int FileLoadFlag = initialize_handler(model_path_str, 0);if (FileLoadFlag >= 0 ){return 0;}return FileLoadFlag;
}
template<typename T> std::vector<float> softmax(const T *logits, unsigned int _size, unsigned int &max_id)
{//types::__assert_type<T>();if (_size == 0 || logits == nullptr) return{};float max_prob = 0.f, total_exp = 0.f;std::vector<float> softmax_probs(_size);for (unsigned int i = 0; i < _size; ++i){softmax_probs[i] = std::exp((float)logits[i]);total_exp += softmax_probs[i];}for (unsigned int i = 0; i < _size; ++i){softmax_probs[i] = softmax_probs[i] / total_exp;if (softmax_probs[i] > max_prob){max_id = i;max_prob = softmax_probs[i];}}return softmax_probs;
}
float SqueezeNetLiveSpoof::detect_handler(const unsigned char* pData, int width, int height, int nchannel, int mod)
{if (!pData || width <= 0 || height <= 0) return 0.0f;try {// 1. 將輸入數據轉換為OpenCV Matcv::Mat input_mat(height, width, nchannel == 3 ? CV_8UC3 : CV_8UC1, (void*)pData);if (nchannel == 1) {cv::cvtColor(input_mat, input_mat, cv::COLOR_GRAY2BGR);}// 2. 預處理圖像this->transform(input_mat);// 3. 運行推理mnn_interpreter->runSession(mnn_session);// 4. 獲取輸出auto output_tensor = mnn_interpreter->getSessionOutput(mnn_session, nullptr);MNN::Tensor host_tensor(output_tensor, output_tensor->getDimensionType());output_tensor->copyToHostTensor(&host_tensor);auto embedding_dims = host_tensor.shape(); // (1,128)const unsigned int hidden_dim = embedding_dims.at(1);const float* embedding_values = host_tensor.host<float>();unsigned int pred_live = 0;auto softmax_probs = softmax<float>(embedding_values, hidden_dim, pred_live);//std::cout << "softmax_probs: " << softmax_probs[0]<<"  "<<softmax_probs[1] << std::endl;float live_score = softmax_probs[0]; // 取真臉概率作為活體分數std::cout << "live_score: " << live_score<< "   spoof_score:"<< softmax_probs[1]<< std::endl;return live_score;} catch (const std::exception& e) {std::cerr << "detect_handler exception: " << e.what() << std::endl;return 0.0f;}
}
void SqueezeNetLiveSpoof::initialize_pretreat() {// 初始化預處理參數MNN::CV::Matrix trans;trans.setScale(1.0f, 1.0f);MNN::CV::ImageProcess::Config img_config;img_config.filterType = MNN::CV::BICUBIC;::memcpy(img_config.mean, meanVals_, sizeof(meanVals_));::memcpy(img_config.normal, normVals_, sizeof(normVals_));img_config.sourceFormat = MNN::CV::BGR;img_config.destFormat = MNN::CV::RGB;pretreat = std::shared_ptr<MNN::CV::ImageProcess>(MNN::CV::ImageProcess::create(img_config));pretreat->setMatrix(trans);
}
void SqueezeNetLiveSpoof::transform(const cv::Mat &mat){cv::Mat mat_rs;cv::resize(mat, mat_rs, cv::Size(input_width, input_height));pretreat->convert(mat_rs.data, input_width, input_height, mat_rs.step[0], input_tensor);
}

四、結果展示

在返回的分類結果中，我們用0.8作為閾值對活體分數進行過濾，得到的結果如下：
判斷為非活體的圖片結果
在這里插入圖片描述

五、留下來的問題

一個從數據整理到最后的MNN推理的2D活體檢測的工作簡單的完結了，這個系列的內容主要目的是講訴一個模型如何從設計到部署的全過程，過程中的有些描述和個人理解并不一定正確，如果有其他理解或者錯處指出，請嚴重指出。
深度學習之用CelebA_Spoof數據集搭建一個活體檢測-數據處理
深度學習之用CelebA_Spoof數據集搭建一個活體檢測-模型搭建和訓練
深度學習之用CelebA_Spoof數據集搭建一個活體檢測-模型驗證與測試
深度學習之用CelebA_Spoof數據集搭建一個活體檢測-一些模型訓練中的改動帶來的改善
那么這個系列完結，留下什么問題：
1 2D活體檢測有沒有更好的方法？
2 訓練的過程如何更好更快的調參以及收斂，以及如何尋找更好的特征？
3 在實際使用過程中，怎樣提高功能的體驗感，至于那些判斷錯誤的，該如何進行處理？
4 如何在不同環境下，保證活體的準確率？
這都是在這個工作中需要重視的，而且這項工作并不會因為有了部署成功就能成功，而是需要不斷改善。如果有好的方法和建議，煩請留言告知，我們一起討論！