Comparison of onnx, openvino and mnn inference speed

Onnx, openvino and mnn can all be used as the framework of cpu reasoning, the reasoning speed is compared here

The python version of openvino used by the model uses the onnx file of this article

Note: The mnn and onnx files used here are not quantized and other operations. They are directly converted and read directly.

The comparison code is as follows:

from openvino.inference_engine import IECore
import onnxruntime
import MNN
import numpy as np
import cv2
import time

model="ctdet_coco_dlav0_512.onnx"
mnnmodel="ctdet_coco_dlav0_512.mnn"

#openvino
ie = IECore()
net = ie.read_network(model=model)
input_blob = next(iter(net.input_info))
out_blob = next(iter(net.outputs))
net.batch_size=1#batchsize
n, c, h, w = net.input_info[input_blob].input_data.shape
print(n, c, h, w)
images = np.ndarray(shape=(n, c, h, w))
for i in range(n):
        image = cv2.imread("123.jpg")
        if image.shape[:-1] != (h, w):
            image = cv2.resize(image, (w, h))
        image = image.transpose((2, 0, 1))
        images[i] = image
images=images.astype(np.float32)
exec_net = ie.load_network(network=net, device_name="CPU")

#onnxruntime
session = onnxruntime.InferenceSession(model)
inputs = {session.get_inputs()[0].name: images.astype(np.float32)}

#mnn
interpreter = MNN.Interpreter(mnnmodel)
mnn_session = interpreter.createSession()
input_tensor = interpreter.getSessionInput(mnn_session)
tmp_input = MNN.Tensor((1, 3, 512, 512),\
MNN.Halide_Type_Float, images[0], MNN.Tensor_DimensionType_Tensorflow)  
input_tensor.copyFrom(tmp_input)

#onnxruntime infer
start=time.time()
out=session.run(None, inputs)
print('onnxruntime infer total time is %.4f s'%(time.time()-start))
#openvino infer
start=time.time()
res = exec_net.infer(inputs={input_blob: images})
#print(res)
print('openvino infer total time is %.4f s'%(time.time()-start))
#mnn infer
start=time.time()
interpreter.runSession(mnn_session)
print('mnn infer total time is %.4f s'%(time.time()-start))

operation result:

Conclusion: It can be seen that openvino does optimize the speed a lot, the improvement is more than 30%, and the speed improvement of mnn is not obvious