基于FPGA的车牌识别系统
这个项目是之前参加"FPGA创新设计邀请赛"时开始写的程序,后来参加学校"罗姆杯"的时候又改进了一下.程序基于Xilinx公司的Pynq-Z2开发板,使用opencv库完成车牌识别.
项目背景和设计目的
• 车牌识别系统是计算机视频图像识别技术在车辆牌照识别中的一种应用，在高速公路、停车场、小区、道路等环境下有着广泛的应用。
• 我们希望能够充分利用PYNQ的内部资源，运用Python语言的程序设计和OpenCV计算机视觉库，设计出一个较为可靠的车牌识别系统，将输出结果显示到显示器上，包含车牌号码和车速等信息。
• 对于停车场门口或收费站等应用场景,本系统还可以直接控制舵机,用于限制车辆的进出
产品特点
• 运用了Python语言的程序设计和OpenCV计算机视觉库，使用了ZYNQ-7020芯片，将ARM处理器嵌入到FPGA中，这样将ARM处理器的优势和FPGA的优势结合到了一起。
• 研究基于FPGA+ARM平台的车牌识别系统可利用ARM处理器在操作系统方面的优势，能够实现良好的人机交互功能；又利用FPGA在并行算法加速、可动态重配置的特点，实现硬件加速、增加系统的灵活性，提高车牌识别速度。
• 使用USB外设单目摄像头读取图像信息，能准确、清晰地搜集到车辆图像，图像分辨率达到设计要求。使用外设显示屏输出显示车牌信息。
• 外接舵机，模拟小区出入口和高速公路收费站。
硬件平台
使用Xilinx公司的PYNQ-Z2开发板，搭载ZYNQ-7020芯片和ARM-A9双核处理，使用Python语言和Verilog HDL硬件描述语言对板子进行描述。
性能和成本
现场可编程门阵列（FPGA）和图形处理器（GPU）的性能对比。FPGA的功耗远小于GPU，并且FPGA可以根据特定的应用去编程硬件，虽然FPGA的运算速度相较GPU较慢，总体上，谁在机器视觉方面具备优势是一个需要深入研究的课题，众多企业对FPGA的应用前景比较看好，具备较强的研究价值和应用前景。
FPGA的造价相对GPU较低，在未来，随着技术的成熟和广泛的应用，FPGA的制造成本和销售价格推测均会进一步降低，在现在和将来能够带来的经济收益会高于GPU，在这方面，FPGA应用于机器视觉领域具备很强的潜在的社会经济效益。
整体框图
FPGA处理模块
算法流程图
运行结果
改进
图像处理的拓展。在本次设计中，尚未加入车型识别/驾驶员特征（如有无系安全带等）等内容的读取，在进一步研究中可以考虑增加上述内容的研究和识别。
加速模块尚未完善,加速效果不显著。
物联网方向的深入。在本次设计中，尚未设置联网比对功能，如果有这部分的设计，那么我们可以实时与公安的数据进行比对，如车辆颜色等特征判断是否为套牌，车牌是否系伪造（数据库中无该车牌），是否是被盗车辆、未缴纳交通强制保险车辆、肇事逃逸车辆等，功能会更加完善、齐全。
Python代码
以下为python代码,但是并不包含KNN模型的数据,完整的工程请看:
https://github.com/chenjianqu/Pynq-LicensePlateRecognition
from pynq.overlays.base import BaseOverlay
from pynq.lib.video import *
from matplotlib import pyplot as plt
import numpy as np
import cv2
import time
import math
from time import sleep
from pynq.lib.pmod import Pmod_PWM
from pynq.lib.pmod import PMODA
base = BaseOverlay("base.bit")
pwm = Pmod_PWM(PMODA, 0)
#读取字典和数据
with open('knn_dict_letter.txt','r') as f:
labelDict = eval(f.read())
with np.load('knn_data.npz') as data:
train_data=data['train_data']
train_labels=data['train_labels']
with np.load('knn_data_zh.npz') as data:
train_data_zh=data['train_data_zh']
train_labels_zh=data['train_labels_zh']
# 定义分类器
cascade_path = 'cascade.xml'
cascade_car_path = 'cars.xml'
car_cascade = cv2.CascadeClassifier(cascade_car_path)
cascade = cv2.CascadeClassifier(cascade_path)
# monitor configuration: 640*480 @ 24Hz
Mode = VideoMode(640,480,24)
hdmi_out = base.video.hdmi_out
hdmi_out.configure(Mode,PIXEL_BGR)
hdmi_out.start()
# monitor (output) frame buffer size
frame_out_w = 640
frame_out_h = 480
scale=400/480 #比例系数
PLATE_WIDTH=0.44
Hmin=100
Hmax=124
Smin=120
Smax=255
Vmin=120
Vmax=255
reg=["ShangHaiF19911","ShangHaiB6N866","NingXia1B6N86","ShangHaiB6N866","ShangHaiB8N866","ShangHaiB0N866","NingXiaB19873","HeNanP8A629"]
isRecognize=3
print('Read Data Done')
def KeyCallBack():
if(base.buttons[0].read()==1):
isRecognize=0
elif(base.buttons[1].read()==1):
isRecognize=1
elif(base.buttons[2].read()==1):
isRecognize=2
elif(base.buttons[3].read()==1):
isRecognize=3
#精确的定位车牌
def locateAccuracy(img):
frame=cv2.cvtColor(img,cv2.COLOR_BGR2HSV)
height = frame.shape[0]
width = frame.shape[1]
top=0
button=height-1
left=0
right=width-1
row=0
while(row<height):
col=0
count=0
while(col<width):
if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):
count+=1
col+=1
if(count/width>0.6):
top=row
break
row+=1
row=button
while(row>0):
col=0
count=0
while(col<width):
if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):
count+=1
col+=1
if(count/width>0.6):
button=row
break
row-=1
col=right
while(col>0):
row=0
count=0
while(row<height):
if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):
count+=1
row+=1
if(count/height>0.6):
right=col
break
col-=1
col=left
while(col<width):
row=0
count=0
while(row<height):
if((frame[row,col,0]>=Hmin and frame[row,col,0]<=Hmax) and (frame[row,col,1]>=Smin and frame[row,col,1]<=Smax) and (frame[row,col,2]>=Vmin and frame[row,col,2]<=Vmax)):
count+=1
row+=1
if(count/height>0.6):
left=col
break
col+=1
return top,button,left,right
def recognize(img):
top,button,left,right=locateAccuracy(img)
image=img[top:button,left:right]
if(image.size==0):
return []
img_gray=cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
img_thre=cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,35,2)#自适应二值化
#对对图像进行垂直投影
arr=np.zeros(image.shape[1])
col=0
while(col<img_thre.shape[1]):
row=0
count=0
while(row<img_thre.shape[0]):
count+=img_thre[row,col]
row+=1
arr[col]=int(count/255)
col+=1
#根据投影结果进行分割字符
count_1=0
flag=0
flag_index=0
i=0
for c in arr:
if(c<10):
count_1+=1
else:
if(count_1>flag):
flag=count_1
flag_index=int(i-count_1/2)
if(count_1>3):
arr[int(i-count_1/2)]=-1
count_1=0
i+=1
i=0
j=0
x=0
y=top
h=button-top
#获得分割结果
charList=[]
for c in arr:
if(c==-1):
w=i-x
charList.append([x+left,y,w,h])
x=i
if(flag_index==c and (j!=1 or j!=2)):
return []
i+=1
j+=1
charList.append([x+left,y,right-x,h])
if(len(charList)<=5 or len(charList)>8):
return []
return charList
def recognizeRect(img_thre,rect,knn):
x,y,w,h=rect
roi=img_thre[y:(y+h),x:(x+w)]
if h>w:
roi=cv2.copyMakeBorder(roi,0,0,int((h-w)/2),int((h-w)/2),cv2.BORDER_CONSTANT,value=[0,0,0])
roi=cv2.resize(roi,(20,20))
#cv2.imshow("char",roi)
#cv2.waitKey(0)
roivector=roi.reshape(1,400).astype(np.float32)
ret,result,neighbours,dist=knn.findNearest(roivector,k=6)#进行预测
return int(ret)
def access_pixels(frame):
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
#print(frame.shape) #shape内包含三个元素：按顺序为高、宽、通道数
height = frame.shape[0]
weight = frame.shape[1]
count=0
for row in range(height): #遍历高
for col in range(weight): #遍历宽
if((frame[row,col,0]>=100 and frame[row,col,0]<=124) and (frame[row,col,1]>=43 and frame[row,col,1]<=255) and (frame[row,col,2]>=46 and frame[row,col,2]<=255)):
count+=1
if(count/(height*weight)>0.5):
return True
else:
return False
def isPlate(frame):
if(frame.shape[1]>frame.shape[0]/2 or frame.shape[1]*5<frame.shape[0]):
return True
else:
return False
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640);
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480);
print("Capture device is open: " + str(cap.isOpened()))
period=20000
duty=6
pwm.generate(period,duty)
time_start=time.time()
time_last=0
i=1
x_last=0
y_last=0
isRecognize=3
str_plate=""
print("start while")
while(cap.isOpened()):
if(base.buttons[0].read()==1):
isRecognize=0
elif(base.buttons[1].read()==1):
isRecognize=1
elif(base.buttons[2].read()==1):
isRecognize=2
elif(base.buttons[3].read()==1):
isRecognize=3
period=20000
duty=6
pwm.generate(period,duty)
str_plate=""
time_start=time.time()
ret, frame = cap.read()
if (ret):
if(isRecognize==3):
outframe = hdmi_out.newframe()
outframe[0:640,0:480,:] = frame[0:640,0:480,:]
hdmi_out.writeframe(outframe)
else:
time_last=time_start
time_start=time.time()
#print("read frame")
image=cv2.resize(frame,(int(640*scale),400))
#print("after new outframe time is:"+str(time.time()-time_start))
img_gray=cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
#print("after cvtgray time is:"+str(time.time()-time_start))
car_plates = cascade.detectMultiScale(img_gray, 1.1, 2, minSize=(36, 9), maxSize=(36 * 40, 9 * 40))
#print("after mul time is:"+str(time.time()-time_start))
for car_plate in car_plates:
x, y, w, h = car_plate
#plate = image[y: y + h, x: x + w]
#if(isPlate(plate)==False):#根据颜色判断是否是正确的车牌区域
# continue
#if(access_pixels(plate)==False):#根据颜色判断是否是正确的车牌区域
# continue
if(isRecognize==0):
plateScale=(w/PLATE_WIDTH)
v=(math.sqrt((x-x_last)*(x-x_last)+(y-y_last)*(y-y_last))/(time_start-time_last)/plateScale)/i
v=int(v*100)/100
x_last=x
y_last=y
i=0
cv2.putText(image,"Vehicle Velocity:"+str(v)+" m/s",(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 0, 255), 2)
i+=1
elif(isRecognize==1):
if(base.buttons[0].read()==1):
isRecognize=0
elif(base.buttons[1].read()==1):
isRecognize=1
elif(base.buttons[2].read()==1):
isRecognize=2
elif(base.buttons[3].read()==1):
isRecognize=3
period=20000
duty=6
pwm.generate(period,duty)
str_plate=""
#分割字符
img_thre=cv2.adaptiveThreshold(img_gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,35,2)#自适应二值化
plate=image[y: y + h, x: x + w]
charRect=recognize(image[y: y + h, x: x + w])
if(len(charRect)==0):
continue
if(len(charRect)>7):
charRect.pop()
str_plate=""
KeyCallBack();
#模型创建
knn_zh=cv2.ml.KNearest_create()
knn_zh.train(train_data_zh,cv2.ml.ROW_SAMPLE,train_labels_zh)
#识别中文
rect=charRect[0]
x1,y1,w1,h1=rect
x1=x+x1
y1=y+y1
str_plate=labelDict[recognizeRect(img_thre,(x1,y1,w1,h1),knn_zh)]
if(len(charRect)>0):
x1,y1,w1,h1=charRect[0]
cv2.rectangle(image,(x1,y1),(x1+w1,y1+h1),(255,0,0),1)
if(base.buttons[0].read()==1):
isRecognize=0
elif(base.buttons[1].read()==1):
isRecognize=1
elif(base.buttons[2].read()==1):
isRecognize=2
elif(base.buttons[3].read()==1):
isRecognize=3
str_plate=""
period=20000
duty=6
pwm.generate(period,duty)
knn=cv2.ml.KNearest_create()
knn.train(train_data,cv2.ml.ROW_SAMPLE,train_labels)
for rect in charRect[1:]:
x1,y1,w1,h1=rect
x1=x+x1
y1=y+y1
cv2.rectangle(image,(x1,y1),(x1+w1,y1+h1),(255,0,0),1)#框出字块
s=labelDict[recognizeRect(img_thre,(x1,y1,w1,h1),knn)]
str_plate=str_plate+s
for re in reg:
if(re in str_plate):
period=20000
duty=11
pwm.generate(period,duty)
print("********************ok")
x1,y1,w1,h1=rect
print('recognize time:',time.time()-time_start,'s')
image=cv2.putText(image,str_plate,(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 0, 255), 2)
elif(isRecognize==2):
image=cv2.putText(image,str_plate,(20,image.shape[0]-10),cv2.FONT_HERSHEY_PLAIN, 2.0, (0, 0, 255), 2)
#标出粗定位的车牌
cv2.rectangle(image, (x - 10, y - 10), (x + w + 10, y + h + 10), (255, 255, 255), 1)
image=cv2.resize(image,(640,480))
outframe = hdmi_out.newframe()
outframe[0:640,0:480,:] = image[0:640,0:480,:]
hdmi_out.writeframe(outframe)
#print("after write time is:"+str(time.time()-time_start))
else:
raise RuntimeError("Failed to read from camera.")
print("end program")
