一、概述
基于 DNN 的人脸检测和识别，用于人脸检测的cv::FaceDetectorYN类和用于人脸识别的cv::FaceRecognizerSF类。此模块有两个模型（ONNX 格式）经过预训练并需要下载模型：
（1）FaceDetectorYN
模型下载地址opencv_zoo/models/face_detection_yunet at master · opencv/opencv_zoo · GitHubModel Zoo For OpenCV DNN and Benchmarks. Contribute to opencv/opencv_zoo development by creating an account on GitHub.
https://github.com/opencv/opencv_zoo/tree/master/models/face_detection_yunet 在WIDER Face验证集的结果：0.830(easy), 0.824(medium), 0.708(hard)。
（2）FaceRecognizerSF
二、OpenCV源码
1、源码路径
opencv/modules/objdetect/src/face_detect.cppopencv/modules/objdetect/src/face_recognize.cpp2、源码代码
以下为部分参考代码
int detect(InputArray input_image, OutputArray faces) override
{
// TODO: more checkings should be done?
if (input_image.empty())
{
return 0;
}
CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size");
// Build blob from input image
Mat input_blob = dnn::blobFromImage(input_image);
// Forward
std::vector<String> output_names = { "loc", "conf", "iou" };
std::vector<Mat> output_blobs;
net.setInput(input_blob);
net.forward(output_blobs, output_names);
// Post process
Mat results = postProcess(output_blobs);
results.convertTo(faces, CV_32FC1);
return 1;
}#ifdef HAVE_OPENCV_DNN
class FaceRecognizerSFImpl : public FaceRecognizerSF
{
public:
FaceRecognizerSFImpl(const String& model, const String& config, int backend_id, int target_id)
{
net = dnn::readNet(model, config);
CV_Assert(!net.empty());
net.setPreferableBackend(backend_id);
net.setPreferableTarget(target_id);
};
void alignCrop(InputArray _src_img, InputArray _face_mat, OutputArray _aligned_img) const override
{
Mat face_mat = _face_mat.getMat();
float src_point[5][2];
for (int row = 0; row < 5; ++row)
{
for(int col = 0; col < 2; ++col)
{
src_point[row][col] = face_mat.at<float>(0, row*2+col+4);
}
}
Mat warp_mat = getSimilarityTransformMatrix(src_point);
warpAffine(_src_img, _aligned_img, warp_mat, Size(112, 112), INTER_LINEAR);
};
void feature(InputArray _aligned_img, OutputArray _face_feature) override
{
Mat inputBolb = dnn::blobFromImage(_aligned_img, 1, Size(112, 112), Scalar(0, 0, 0), true, false);
net.setInput(inputBolb);
net.forward(_face_feature);
};
double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type) const override
{
Mat face_feature1 = _face_feature1.getMat(), face_feature2 = _face_feature2.getMat();
normalize(face_feature1, face_feature1);
normalize(face_feature2, face_feature2);
if(dis_type == DisType::FR_COSINE){
return sum(face_feature1.mul(face_feature2))[0];
}else if(dis_type == DisType::FR_NORM_L2){
return norm(face_feature1, face_feature2);
}else{
throw std::invalid_argument("invalid parameter " + std::to_string(dis_type));
}
};
private:
Mat getSimilarityTransformMatrix(float src[5][2]) const {
float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} };
float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5;
float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5;
//Compute mean of src and dst.
float src_mean[2] = { avg0, avg1 };
float dst_mean[2] = { 56.0262f, 71.9008f };
//Subtract mean from src and dst.
float src_demean[5][2];
for (int i = 0; i < 2; i++)
{
for (int j = 0; j < 5; j++)
{
src_demean[j][i] = src[j][i] - src_mean[i];
}
}
float dst_demean[5][2];
for (int i = 0; i < 2; i++)
{
for (int j = 0; j < 5; j++)
{
dst_demean[j][i] = dst[j][i] - dst_mean[i];
}
}
double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0;
for (int i = 0; i < 5; i++)
A00 += dst_demean[i][0] * src_demean[i][0];
A00 = A00 / 5;
for (int i = 0; i < 5; i++)
A01 += dst_demean[i][0] * src_demean[i][1];
A01 = A01 / 5;
for (int i = 0; i < 5; i++)
A10 += dst_demean[i][1] * src_demean[i][0];
A10 = A10 / 5;
for (int i = 0; i < 5; i++)
A11 += dst_demean[i][1] * src_demean[i][1];
A11 = A11 / 5;
Mat A = (Mat_<double>(2, 2) << A00, A01, A10, A11);
double d[2] = { 1.0, 1.0 };
double detA = A00 * A11 - A01 * A10;
if (detA < 0)
d[1] = -1;
double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} };
Mat s, u, vt, v;
SVD::compute(A, s, u, vt);
double smax = s.ptr<double>(0)[0]>s.ptr<double>(1)[0] ? s.ptr<double>(0)[0] : s.ptr<double>(1)[0];
double tol = smax * 2 * FLT_MIN;
int rank = 0;
if (s.ptr<double>(0)[0]>tol)
rank += 1;
if (s.ptr<double>(1)[0]>tol)
rank += 1;
double arr_u[2][2] = { {u.ptr<double>(0)[0], u.ptr<double>(0)[1]}, {u.ptr<double>(1)[0], u.ptr<double>(1)[1]} };
double arr_vt[2][2] = { {vt.ptr<double>(0)[0], vt.ptr<double>(0)[1]}, {vt.ptr<double>(1)[0], vt.ptr<double>(1)[1]} };
double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0];
double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0];
if (rank == 1)
{
if ((det_u*det_vt) > 0)
{
Mat uvt = u*vt;
T[0][0] = uvt.ptr<double>(0)[0];
T[0][1] = uvt.ptr<double>(0)[1];
T[1][0] = uvt.ptr<double>(1)[0];
T[1][1] = uvt.ptr<double>(1)[1];
}
else
{
double temp = d[1];
d[1] = -1;
Mat D = (Mat_<double>(2, 2) << d[0], 0.0, 0.0, d[1]);
Mat Dvt = D*vt;
Mat uDvt = u*Dvt;
T[0][0] = uDvt.ptr<double>(0)[0];
T[0][1] = uDvt.ptr<double>(0)[1];
T[1][0] = uDvt.ptr<double>(1)[0];
T[1][1] = uDvt.ptr<double>(1)[1];
d[1] = temp;
}
}
else
{
Mat D = (Mat_<double>(2, 2) << d[0], 0.0, 0.0, d[1]);
Mat Dvt = D*vt;
Mat uDvt = u*Dvt;
T[0][0] = uDvt.ptr<double>(0)[0];
T[0][1] = uDvt.ptr<double>(0)[1];
T[1][0] = uDvt.ptr<double>(1)[0];
T[1][1] = uDvt.ptr<double>(1)[1];
}
double var1 = 0.0;
for (int i = 0; i < 5; i++)
var1 += src_demean[i][0] * src_demean[i][0];
var1 = var1 / 5;
double var2 = 0.0;
for (int i = 0; i < 5; i++)
var2 += src_demean[i][1] * src_demean[i][1];
var2 = var2 / 5;
double scale = 1.0 / (var1 + var2)* (s.ptr<double>(0)[0] * d[0] + s.ptr<double>(1)[0] * d[1]);
double TS[2];
TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1];
TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1];
T[0][2] = dst_mean[0] - scale*TS[0];
T[1][2] = dst_mean[1] - scale*TS[1];
T[0][0] *= scale;
T[0][1] *= scale;
T[1][0] *= scale;
T[1][1] *= scale;
Mat transform_mat = (Mat_<double>(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]);
return transform_mat;
}
private:
dnn::Net net;
};
#endif
Ptr<FaceRecognizerSF> FaceRecognizerSF::create(const String& model, const String& config, int backend_id, int target_id)
{
#ifdef HAVE_OPENCV_DNN
return makePtr<FaceRecognizerSFImpl>(model, config, backend_id, target_id);
#else
CV_UNUSED(model); CV_UNUSED(config); CV_UNUSED(backend_id); CV_UNUSED(target_id);
CV_Error(cv::Error::StsNotImplemented, "cv::FaceRecognizerSF requires enabled 'dnn' module");
#endif
}三、示例参考
1、完整代码(来自opencv官方)
#include <opencv2/dnn.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/objdetect.hpp>
#include <iostream>
using namespace cv;
using namespace std;
static
void visualize(Mat& input, int frame, Mat& faces, double fps, int thickness = 2)
{
std::string fpsString = cv::format("FPS : %.2f", (float)fps);
if (frame >= 0)
cout << "Frame " << frame << ", ";
cout << "FPS: " << fpsString << endl;
for (int i = 0; i < faces.rows; i++)
{
// Print results
cout << "Face " << i
<< ", top-left coordinates: (" << faces.at<float>(i, 0) << ", " << faces.at<float>(i, 1) << "), "
<< "box width: " << faces.at<float>(i, 2) << ", box height: " << faces.at<float>(i, 3) << ", "
<< "score: " << cv::format("%.2f", faces.at<float>(i, 14))
<< endl;
// Draw bounding box
rectangle(input, Rect2i(int(faces.at<float>(i, 0)), int(faces.at<float>(i, 1)), int(faces.at<float>(i, 2)), int(faces.at<float>(i, 3))), Scalar(0, 255, 0), thickness);
// Draw landmarks
circle(input, Point2i(int(faces.at<float>(i, 4)), int(faces.at<float>(i, 5))), 2, Scalar(255, 0, 0), thickness);
circle(input, Point2i(int(faces.at<float>(i, 6)), int(faces.at<float>(i, 7))), 2, Scalar(0, 0, 255), thickness);
circle(input, Point2i(int(faces.at<float>(i, 8)), int(faces.at<float>(i, 9))), 2, Scalar(0, 255, 0), thickness);
circle(input, Point2i(int(faces.at<float>(i, 10)), int(faces.at<float>(i, 11))), 2, Scalar(255, 0, 255), thickness);
circle(input, Point2i(int(faces.at<float>(i, 12)), int(faces.at<float>(i, 13))), 2, Scalar(0, 255, 255), thickness);
}
putText(input, fpsString, Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0), 2);
}
int main(int argc, char** argv)
{
CommandLineParser parser(argc, argv,
"{help h | | Print this message}"
"{image1 i1 | | Path to the input image1. Omit for detecting through VideoCapture}"
"{image2 i2 | | Path to the input image2. When image1 and image2 parameters given then the program try to find a face on both images and runs face recognition algorithm}"
"{video v | 0 | Path to the input video}"
"{scale sc | 1.0 | Scale factor used to resize input video frames}"
"{fd_model fd | face_detection_yunet_2021dec.onnx| Path to the model. Download yunet.onnx in https://github.com/opencv/opencv_zoo/tree/master/models/face_detection_yunet}"
"{fr_model fr | face_recognition_sface_2021dec.onnx | Path to the face recognition model. Download the model at https://github.com/opencv/opencv_zoo/tree/master/models/face_recognition_sface}"
"{score_threshold | 0.9 | Filter out faces of score < score_threshold}"
"{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold}"
"{top_k | 5000 | Keep top_k bounding boxes before NMS}"
"{save s | false | Set true to save results. This flag is invalid when using camera}"
);
if (parser.has("help"))
{
parser.printMessage();
return 0;
}
String fd_modelPath = parser.get<String>("fd_model");
String fr_modelPath = parser.get<String>("fr_model");
float scoreThreshold = parser.get<float>("score_threshold");
float nmsThreshold = parser.get<float>("nms_threshold");
int topK = parser.get<int>("top_k");
bool save = parser.get<bool>("save");
float scale = parser.get<float>("scale");
double cosine_similar_thresh = 0.363;
double l2norm_similar_thresh = 1.128;
// Initialize FaceDetectorYN
Ptr<FaceDetectorYN> detector = FaceDetectorYN::create(fd_modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK);
TickMeter tm;
// If input is an image
if (parser.has("image1"))
{
String input1 = parser.get<String>("image1");
Mat image1 = imread(samples::findFile(input1));
if (image1.empty())
{
std::cerr << "Cannot read image: " << input1 << std::endl;
return 2;
}
int imageWidth = int(image1.cols * scale);
int imageHeight = int(image1.rows * scale);
resize(image1, image1, Size(imageWidth, imageHeight));
tm.start();
// Set input size before inference
detector->setInputSize(image1.size());
Mat faces1;
detector->detect(image1, faces1);
if (faces1.rows < 1)
{
std::cerr << "Cannot find a face in " << input1 << std::endl;
return 1;
}
tm.stop();
// Draw results on the input image
visualize(image1, -1, faces1, tm.getFPS());
// Save results if save is true
if (save)
{
cout << "Saving result.jpg...\n";
imwrite("result.jpg", image1);
}
// Visualize results
imshow("image1", image1);
pollKey(); // handle UI events to show content
if (parser.has("image2"))
{
String input2 = parser.get<String>("image2");
Mat image2 = imread(samples::findFile(input2));
if (image2.empty())
{
std::cerr << "Cannot read image2: " << input2 << std::endl;
return 2;
}
tm.reset();
tm.start();
detector->setInputSize(image2.size());
Mat faces2;
detector->detect(image2, faces2);
if (faces2.rows < 1)
{
std::cerr << "Cannot find a face in " << input2 << std::endl;
return 1;
}
tm.stop();
visualize(image2, -1, faces2, tm.getFPS());
if (save)
{
cout << "Saving result2.jpg...\n";
imwrite("result2.jpg", image2);
}
imshow("image2", image2);
pollKey();
// Initialize FaceRecognizerSF
Ptr<FaceRecognizerSF> faceRecognizer = FaceRecognizerSF::create(fr_modelPath, "");
// Aligning and cropping facial image through the first face of faces detected.
Mat aligned_face1, aligned_face2;
faceRecognizer->alignCrop(image1, faces1.row(0), aligned_face1);
faceRecognizer->alignCrop(image2, faces2.row(0), aligned_face2);
// Run feature extraction with given aligned_face
Mat feature1, feature2;
faceRecognizer->feature(aligned_face1, feature1);
feature1 = feature1.clone();
faceRecognizer->feature(aligned_face2, feature2);
feature2 = feature2.clone();
double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE);
double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2);
if (cos_score >= cosine_similar_thresh)
{
std::cout << "They have the same identity;";
}
else
{
std::cout << "They have different identities;";
}
std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n";
if (L2_score <= l2norm_similar_thresh)
{
std::cout << "They have the same identity;";
}
else
{
std::cout << "They have different identities.";
}
std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n";
}
cout << "Press any key to exit..." << endl;
waitKey(0);
}
else
{
int frameWidth, frameHeight;
VideoCapture capture;
std::string video = parser.get<string>("video");
if (video.size() == 1 && isdigit(video[0]))
capture.open(parser.get<int>("video"));
else
capture.open(samples::findFileOrKeep(video)); // keep GStreamer pipelines
if (capture.isOpened())
{
frameWidth = int(capture.get(CAP_PROP_FRAME_WIDTH) * scale);
frameHeight = int(capture.get(CAP_PROP_FRAME_HEIGHT) * scale);
cout << "Video " << video
<< ": width=" << frameWidth
<< ", height=" << frameHeight
<< endl;
}
else
{
cout << "Could not initialize video capturing: " << video << "\n";
return 1;
}
detector->setInputSize(Size(frameWidth, frameHeight));
cout << "Press 'SPACE' to save frame, any other key to exit..." << endl;
int nFrame = 0;
for (;;)
{
// Get frame
Mat frame;
if (!capture.read(frame))
{
cerr << "Can't grab frame! Stop\n";
break;
}
resize(frame, frame, Size(frameWidth, frameHeight));
// Inference
Mat faces;
tm.start();
detector->detect(frame, faces);
tm.stop();
Mat result = frame.clone();
// Draw results on the input image
visualize(result, nFrame, faces, tm.getFPS());
// Visualize results
imshow("Live", result);
int key = waitKey(1);
bool saveFrame = save;
if (key == ' ')
{
saveFrame = true;
key = 0; // handled
}
if (saveFrame)
{
std::string frame_name = cv::format("frame_%05d.png", nFrame);
std::string result_name = cv::format("result_%05d.jpg", nFrame);
cout << "Saving '" << frame_name << "' and '" << result_name << "' ...\n";
imwrite(frame_name, frame);
imwrite(result_name, result);
}
++nFrame;
if (key > 0)
break;
}
cout << "Processed " << nFrame << " frames" << endl;
}
cout << "Done." << endl;
return 0;
}2、代码解释
（1）Initialize FaceDetectorYN
Ptr<FaceDetectorYN> detector = FaceDetectorYN::create(fd_modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK);（2）在推理之前设置输入大小
// Set input size before inference
detector->setInputSize(image1.size());
Mat faces1;
detector->detect(image1, faces1);
if (faces1.rows < 1)
{
std::cerr << "Cannot find a face in " << input1 << std::endl;
return 1;
}检测输出人脸是一个 CV_32F 类型的二维数组，其行是检测到的人脸实例，列是人脸的位置和 5 个人脸地标。 每行的格式如下：
x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm其中x1, y1, w, h是人脸边界框的左上角坐标，宽高，{x, y}_{re, le, nt, rcm, lcm}代表右眼坐标，左眼坐标 分别是眼睛、鼻尖、右嘴角和左嘴角。
（3）人脸识别
在人脸检测之后，运行以下代码以从人脸图像中提取人脸特征。
// Initialize FaceRecognizerSF
Ptr<FaceRecognizerSF> faceRecognizer = FaceRecognizerSF::create(fr_modelPath, "");
// Aligning and cropping facial image through the first face of faces detected.
Mat aligned_face1, aligned_face2;
faceRecognizer->alignCrop(image1, faces1.row(0), aligned_face1);
faceRecognizer->alignCrop(image2, faces2.row(0), aligned_face2);
// Run feature extraction with given aligned_face
Mat feature1, feature2;
faceRecognizer->feature(aligned_face1, feature1);
feature1 = feature1.clone();
faceRecognizer->feature(aligned_face2, feature2);
feature2 = feature2.clone();获取两张人脸图像的人脸特征 feature1 和 feature2 后，运行以下代码计算两张人脸的身份差异。
double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE);
double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2);例如，如果余弦距离大于或等于 0.363，或者 normL2 距离小于或等于 1.128，则两个人脸有相同的身份。