This article and the following are used for the learning of license plate recognition of the open source project OpenCV! ! !
The process of license plate recognition can be divided into four steps: 1) license plate image cutting; 2) license plate image classification; 3) license plate character cutting; 4) license plate character classification;
1) License plate image cutting:
a. Convert the original image into a grayscale image, which can remove the external noise generated by multi-channel;
b. sobel filtering, an important feature of license plate segmentation is that there are many vertical edges in the license plate. In order to extract vertical edges, the first-order vertical derivative of sobel is used;
c. Thresholding, applying an OSTU method to automatically obtain a threshold filter to obtain a binary image;
d. Close operation morphology, connect all areas with a large number of edges, delete the blank areas between the edges, and connect the license plate areas;
e. Filled with water, all license plates have a uniform background color. Use the flood fill algorithm to get an accurate trim of the rotation matrix. The flood fill function fills connected regions to the mask image with color, starting from the seed. The filled pixels are compared with the seed points. If the pixel value is x, seed-low<=x<=seed+up, the position will be filled. Once the mask image for clipping is obtained, and thus the smallest bounding rectangle of the mask image points is obtained, check the rectangle size again. For each mask, the white pixel obtains the position and uses the minAreaRect function to retrieve the closest trimmed area;
f. Affine transformation, used to remove the rotation of the detected rectangular area;
g. Extract the rectangle;
h. Resized to a uniform size, histogram equalized, the extracted rectangular images cannot be used well for training and classification because they do not have the same size. Also, each image contains different lighting conditions, increasing the difference between them.
/* imageSlicer.cpp */
#include <iostream>
#include <opencv2/opencv.hpp>
#include <time.h>
using namespace std;
using namespace cv;
//Filter the area and aspect ratio of the rotated rectangle
bool areaDetection(RotatedRect rectArea)
{
float error = 0.4;//Allow error rate
const float width_height = 4.7272;//Get the aspect ratio of the Spanish license plate 57200
int min_area = 25 * 25 * width_height;//Get the maximum and minimum area of the allowed rectangle
int max_area = 100 * 100 * width_height;
float min_value = width_height*(1 - error);//Get the maximum and minimum aspect ratio of the allowed rectangle 1.890-6.6180
float max_value = width_height*(1 + error);
int rect_area = rectArea.size.width*rectArea.size.height;//Calculate the enclosing area and aspect ratio of the rotatable rectangle
float rect_value = rectArea.size.width / rectArea.size.height;
rect_value = rect_value<1 ? 1 / rect_value : rect_value;
return rect_area>min_area && rect_area<max_area && rect_value>min_value && rect_value<max_value;
}
void imgProcess(Mat carImg)
{
//1. Get the grayscale image
Mat grayImg, blurImg, sobelImg, threshImg, mopImg;
cvtColor(carImg, grayImg, COLOR_BGR2GRAY);
//imshow("Grayscale", grayImg);
//2. Mean filter
/*void blur(InputArray src, OutputArray dst, Size ksize, Point anchor=Point(-1,-1),
int borderType=BORDER_DEFAULT )*/
blur(grayImg, blurImg, Size(5, 5));//Mean filter
//imshow("Mean filter", blurImg);
//3.sobel edge detection
/*void Sobel(InputArray src, OutputArray dst, int ddepth, int dx, int dy, int ksize=3, double scale=1,
double delta=0, int borderType=BORDER_DEFAULT )*/
Sobel(blurImg, sobelImg, CV_8U, 1, 0);//Sobel operator edge detection only seeks the first derivative for the x direction
//imshow("sobel edge detection", sobelImg);
//4. Binarization, the OTSU algorithm automatically obtains the threshold to obtain the binarized image
/*double threshold(InputArray src, OutputArray dst, double thresh, double maxval, int type)*/
threshold(sobelImg, threshImg, 0, 255, CV_THRESH_OTSU+CV_THRESH_BINARY);
//imshow("Image binarization", threshImg);
//5. Close operation, delete the blank area between the edges, and connect the license plate area
/*Mat getStructuringElement(int shape, Size ksize, Point anchor=Point(-1,-1))*/
Mat element = getStructuringElement(MORPH_RECT, Size(17, 3));//Define structural elements
/*void morphologyEx(InputArray src, OutputArray dst, int op, InputArray kernel, Point anchor=Point(-1,-1),
int iterations=1, int borderType=BORDER_CONSTANT, const Scalar& borderValue=morphologyDefaultBorderValue() )*/
morphologyEx(threshImg, mopImg, CV_MOP_CLOSE, element);
//imshow("Closed operation", mopImg);
//6. Find all contours, then filter by area and aspect ratio
vector<vector<Point>> contours;//Define contours, each contour is a set of points
/*void findContours(InputOutputArray Img, OutputArrayOfArrays contours, OutputArray hierarchy,
int mode, int method, Point offset=Point())*/
findContours(mopImg, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);//Detect the outermost contour
map<int, RotatedRect> mapCounter;//Define the map container to store the filtered rectangle outline
for (int i = 0; i < contours.size(); ++i)
{
/*void drawContours(InputOutputArray Img, InputArrayOfArrays contours, int contourIdx, const Scalar& color,
int thickness=1, int lineType=8, InputArray hierarchy=noArray(), int maxLevel=INT_MAX, Point offset=Point() )*/
drawContours(mopImg, contours, i, Scalar(255), 1);//Draw the contour of the image
/*RotatedRect minAreaRect(InputArray points)*/
RotatedRect contourRect = minAreaRect(contours[i]);//Given a 2D point set, find the enclosing rectangle of the minimum area point set
Point2f vertices[4];//Define the four vertices of the rectangle
contourRect.points(vertices);
for (int j = 0; j < 4; ++j)
{
line(mopImg, vertices[j], vertices[(j + 1) % 4], Scalar(255), 2);//Draw a rectangle surrounding the outline
}
if (areaDetection(contourRect))
mapCounter[i] = contourRect;
}
//imshow("Rectangle surrounds all contours", mopImg);
cout << "轮廓总数: " << contours.size() << endl;
cout << "Number of contours after filtering:" << mapCounter.size() << endl;
//Get the starting address of the rectangular outline after map storage and filtering for traversal, and continue to filter
map<int, RotatedRect>::iterator it = mapCounter.begin();
while (it != mapCounter.end())
{
//7. Draw a rectangle filtered by area and aspect ratio
//RotatedRect contourRect = it->second;//Get the value corresponding to the key
//Point2f vertices[4];//Define four vertices
//contourRect.points(vertices);
//for (int j = 0; j < 4; ++j)
//{
// line(mopImg, vertices[j], vertices[(j + 1) % 4], Scalar(255), 5);
//}
//++it;
//8. Fill the rectangular outline with water
RotatedRect contourRect = it->second;//Get the value corresponding to the key
/*void circle(Mat& img, Point center, int radius, const Scalar& color,
int thickness=1, int lineType=8, int shift=0)*/
circle(mopImg, contourRect.center, 3, Scalar(255), -1);//Draw the center of the rectangle
float minSize = (contourRect.size.width < contourRect.size.height) ? contourRect.size.width : contourRect.size.height;
minSize = minSize*0.5;//Set the range of random seeds
srand(time(NULL));
int seedNum = 10;//Set the random seed number for each mask image
Mat mask;//Draw a new mask image
mask.create(carImg.rows + 2, carImg.cols + 2, CV_8UC1);
mask = Scalar::all(0);
Scalar newVal = Scalar::all(255);//Fill color
Rect ccomp;//Redraw the minimum rectangular area
Scalar loDiff(30, 30, 30);//Maximum value of brightness or color negative difference
Scalar upDiff(30, 30, 30);//Maximum value of brightness or color positive difference
//Only consider the pixels in the horizontal and vertical directions of the current pixel + the white fill value is 255 (ignore newVal) + consider the difference between the current pixel and the seed pixel + fill the mask image
int flags = 4 + (255 << 8) + FLOODFILL_FIXED_RANGE + FLOODFILL_MASK_ONLY;
for (int j = 0; j < seedNum; ++j)
{
/*int floodFill(InputOutputArray Img, InputOutputArray mask, Point seedPoint, Scalar newVal,
Rect* rect=0, Scalar loDiff=Scalar(), Scalar upDiff=Scalar(), int flags=4 )*/
Point point;
point.x = contourRect.center.x + rand() % (int)minSize - (int)(minSize / 2);//Get the point near the center of the rectangular area as a seed (yellow)
point.y = contourRect.center.y + rand() % (int)minSize - (int)(minSize / 2);
circle(mopImg, point, 1, Scalar(255), -1);//Draw the seed
int area = floodFill(carImg, mask, point, newVal, &ccomp, loDiff, upDiff, flags);
}
//imshow("Flood fill image", mopImg);
//imshow("mask image", mask);
//9. Get the bounding rectangle with the smallest area by each image mask
vector<Point> pointInterest;
Mat_<uchar>::iterator itMask = mask.begin<uchar>();//Mat's lightweight data type Mat_, the type needs to be specified
for (; itMask != mask.end<uchar>(); ++itMask)
{
if (*itMask == 255)
pointInterest.push_back(itMask.pos());//Save the coordinates of the white point
}
RotatedRect minRect = minAreaRect(pointInterest);//Given a 2D point set, find the enclosing rectangle with the smallest area
if (areaDetection(minRect))
{
//10. Draw a rectangle by continuing to filter
Point2f minRectPoints[4];
minRect.points(minRectPoints);
for (int k = 0; k < 4; k++)
line(mopImg, minRectPoints[k], minRectPoints[(k + 1) % 4], Scalar(255));
//11. Affine transformation of the original image
float width_height = (float)minRect.size.width / (float)minRect.size.height;
float angle = minRect.angle;
if (width_height < 1)//Process the license plate with a rotation angle greater than 90 degrees in the image
angle = angle + 90;
Mat rotMat = getRotationMatrix2D(minRect.center, angle, 1);//Get the rotation matrix of the rectangle
Mat warpImg;
/*void warpAffine(InputArray src, OutputArray dst, InputArray M, Size dsize,
int flags=INTER_LINEAR, int borderMode=BORDER_CONSTANT, const Scalar& borderValue=Scalar())*/
warpAffine(carImg, warpImg, rotMat, carImg.size(), INTER_CUBIC);
//imshow("Original image affine transformation", warpImg);
//12. Image cutting
Size minRectSize = minRect.size;
if (width_height < 1)
swap(minRectSize.width, minRectSize.height);
Mat plateImg;
/*void getRectSubPix(InputArray Img, Size patchSize, Point2f center,
OutputArray patch, int patchType=-1 )*/
getRectSubPix(warpImg, minRectSize, minRect.center, plateImg);
//imshow("Original license plate", plateImg);
//13. Adjust the license plate image size to standard 33*144
Mat resizeImg;
resizeImg.create(33, 144, CV_8UC3);
/*void resize(InputArray src, OutputArray dst, Size dsize, double fx=0,
double fy=0, int interpolation=INTER_LINEAR )*/
resize(plateImg, resizeImg, resizeImg.size(), 0, 0, INTER_CUBIC);
imshow("33*144 license plate", resizeImg);
imwrite("plateOri.jpg", resizeImg);
//14. Histogram equalization
Mat histImg;
cvtColor(resizeImg, histImg, COLOR_BGR2GRAY);
blur(histImg, histImg, Size(3, 3));
equalizeHist(histImg, histImg);
imshow("Histogram equalization license plate", histImg);
imwrite("plate.jpg", histImg);
}
++it;
}
}
2) License plate image classification
a. SVM
/* imageClassification.cpp */
#include <iostream>
#include <opencv2/opencv.hpp>
using namespace std;
using namespace cv;
using namespace ml;
void SVMClassifier(Mat srcImg)
{
Mat plateImg;
srcImg.convertTo(plateImg, CV_32FC1);
plateImg = plateImg.reshape(1, 1);//Convert the image into 1 row and m column features, and automatically calculate the number of columns
FileStorage fs;
fs.open("SVM.xml", FileStorage::READ);//Open the xml file
//1. Get training data
Mat trainMat, classesMat;
fs["TrainingData"] >> trainMat;//Read the training dataset and class labels
fs["classes"] >> classesMat;
Ptr<TrainData> trainData = TrainData::create(trainMat,ROW_SAMPLE,classesMat);
//2. Create a classifier and set parameters
SVM::ParamTypes param;//Set SVM parameters
SVM::KernelTypes kernel = SVM::LINEAR;
Ptr<SVM> svm = SVM::create();
svm->setKernel(kernel);
//3. Train the classifier
svm-> trainAuto (trainData);
//4. Prediction
int result = svm->predict(plateImg);
cout << "Prediction result: " << result << endl;
}