TensorFlow于c/c++端的编译部署
TensorFlow编译成独立的so包，用于平台直接调用
为了对在线环境部署的已有代码中嵌入TensorFlow模块，用于引擎调用TensorFlow相关功能，将tf2.0编译成独立.so包，通过接口调用该包所包含的功能。
0. 配置过程
如图所示，整个配置过程主要有四部分组成。1.系统环境说明；2. TensorFlow源码编译；3. Python模型训练与保存；4. C++调用训练好的模型与参数。
1. 系统环境说明
- 整个配置过程中的系统为MacOS系统，TensorFlow版本为2.0，Python版本为2.7；
2. TensorFlow源码编译
2.1.配置系统环境： 安装HomeBrew, bazel, protobuf和eigen；
2.2. 下载并编译tensorflow源码
- 首先在GitHub下载TensorFlow源码，并选择对应安装的版本号。
- 对下载后的TensorFlow源码进行配置
- 编译源码并生成.so文件
3. 模型训练与保存
3.1.Python 训练TensorFlow模型
训练模型得到.pb文件，以此保存模型中的参数，简单线性模型代码：
#coding:utf-8
#python 2.7
import tensorflow as tf
import numpy as np
import os
tf.app.flags.DEFINE_integer('training_iteration', 1000, 'number of training iterations')
tf.app.flags.DEFINE_integer('model_version', 1, 'version number of the model')
tf.app.flags.DEFINE_string('work_dir', 'model/', 'Working directory')
FLAGS = tf.app.flags.FLAGS
sess = tf.InteractiveSession()
x = tf.placeholder('float', shape=[None, 5], name='inputs')
y_ = tf.placeholder('float', shape=[None, 1])
w = tf.get_variable('w', shape=[5, 1], initializer=tf.truncated_normal_initializer)
b = tf.get_variable('b', shape=[1], initializer=tf.zeros_initializer)
sess.run(tf.global_variables_initializer())
y = tf.add(tf.matmul(x, w), b, name='outputs')
ms_loss = tf.reduce_mean((y - y_) ** 2)
train_step = tf.train.GradientDescentOptimizer(0.005).minimize(ms_loss)
train_x = np.random.randn(1000, 5)
#let the model learn the equation of y = x1 *1 +x2 *2 + ...x5 *5
train_y = np.sum(train_x * np.array([1, 2, 3, 4, 5]) + np.random.randn(1000, 5) / 100, axis=1).reshape(-1, 1)
for i in range(FLAGS.training_iteration):
loss, _ = sess.run([ms_loss, train_step], feed_dict={x: train_x, y_: train_y})
if i % 100==0:
print ("loss is:",loss)
graph = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ["inputs", "outputs"])
tf.train.write_graph(graph, ".", FLAGS.work_dir + "liner.pb", as_text=False)
print ("Done exporting!")
- 注意这里一定要把需要输入和输出的变量要以string形式的name在tf.graph_util.convert_variables_to_constants中进行保存，比如说这里的inputs和outputs。得到一个后缀为pb的文件
3.2.加载模型参数
通过以下代码加载该模型，验证模型的正确性。
import tensorflow as tf
import numpy as np
logdir = './model/'
output_graph_path = logdir + 'liner.pb'
with tf.Graph().as_default():
output_graph_def = tf.GraphDef()
with open(output_graph_path, "rb") as f:
output_graph_def.ParseFromString(f.read())
_ = tf.import_graph_def(output_graph_def, name="")
with tf.Session() as sess:
input = sess.graph.get_tensor_by_name("inputs:0")
output = sess.graph.get_tensor_by_name("outputs:0")
result = sess.run(output, feed_dict={input: np.reshape([1.0, 1.0, 1.0, 1.0, 1.0], [-1, 5])})
print (result)
4. C++调用训练好的模型及参数
参考博客
C++源码，一共四个文件，分别是model_loader_base.h、ann_model_loader.h、ann_model_loader.cpp和main.cpp。
model_loader_base.h 文件
#ifndef CPPTENSORFLOW_MODEL_LOADER_BASE_H
#define CPPTENSORFLOW_MODEL_LOADER_BASE_H
#include <iostream>
#include <vector>
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
using namespace tensorflow;
namespace tf_model {
/**
* Base Class for feature adapter, common interface convert input format to tensors
* */
class FeatureAdapterBase{
public:
FeatureAdapterBase() {};
virtual ~FeatureAdapterBase() {};
virtual void assign(std::string, std::vector<double>*) = 0; // tensor_name, tensor_double_vector
std::vector<std::pair<std::string, tensorflow::Tensor> > input;
};
class ModelLoaderBase {
public:
ModelLoaderBase() {};
virtual ~ModelLoaderBase() {};
virtual int load(tensorflow::Session*, const std::string) = 0; //pure virutal function load method
virtual int predict(tensorflow::Session*, const FeatureAdapterBase&, const std::string, double*) = 0;
tensorflow::GraphDef graphdef; //Graph Definition for current model
};
}
#endif //CPPTENSORFLOW_MODEL_LOADER_BASE_H
ann_model_loader.h文件
#ifndef CPPTENSORFLOW_ANN_MODEL_LOADER_H
#define CPPTENSORFLOW_ANN_MODEL_LOADER_H
#include "model_loader_base.h"
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
using namespace tensorflow;
namespace tf_model {
/**
* @brief: Model Loader for Feed Forward Neural Network
* */
class ANNFeatureAdapter: public FeatureAdapterBase {
public:
ANNFeatureAdapter();
~ANNFeatureAdapter();
void assign(std::string tname, std::vector<double>*) override; // (tensor_name, tensor)
};
class ANNModelLoader: public ModelLoaderBase {
public:
ANNModelLoader();
~ANNModelLoader();
int load(tensorflow::Session*, const std::string) override; //Load graph file and new session
int predict(tensorflow::Session*, const FeatureAdapterBase&, const std::string, double*) override;
};
}
#endif //CPPTENSORFLOW_ANN_MODEL_LOADER_H
ann_model_loader.cpp 文件
#include <iostream>
#include <vector>
#include <map>
#include "ann_model_loader.h"
//#include <tensor_shape.h>
using namespace tensorflow;
namespace tf_model {
/**
* ANNFeatureAdapter Implementation
* */
ANNFeatureAdapter::ANNFeatureAdapter() {
}
ANNFeatureAdapter::~ANNFeatureAdapter() {
}
/*
* @brief: Feature Adapter: convert 1-D double vector to Tensor, shape [1, ndim]
* @param: std::string tname, tensor name;
* @parma: std::vector<double>*, input vector;
* */
void ANNFeatureAdapter::assign(std::string tname, std::vector<double>* vec) {
//Convert input 1-D double vector to Tensor
int ndim = vec->size();
if (ndim == 0) {
std::cout << "WARNING: Input Vec size is 0 ..." << std::endl;
return;
}
// Create New tensor and set value
Tensor x(tensorflow::DT_FLOAT, tensorflow::TensorShape({1, ndim})); // New Tensor shape [1, ndim]
auto x_map = x.tensor<float, 2>();
for (int j = 0; j < ndim; j++) {
x_map(0, j) = (*vec)[j];
}
// Append <tname, Tensor> to input
input.push_back(std::pair<std::string, tensorflow::Tensor>(tname, x));
}
/**
* ANN Model Loader Implementation
* */
ANNModelLoader::ANNModelLoader() {
}
ANNModelLoader::~ANNModelLoader() {
}
/**
* @brief: load the graph and add to Session
* @param: Session* session, add the graph to the session
* @param: model_path absolute path to exported protobuf file *.pb
* */
int ANNModelLoader::load(tensorflow::Session* session, const std::string model_path) {
//Read the pb file into the grapgdef member
tensorflow::Status status_load = ReadBinaryProto(Env::Default(), model_path, &graphdef);
if (!status_load.ok()) {
std::cout << "ERROR: Loading model failed..." << model_path << std::endl;
std::cout << status_load.ToString() << "\n";
return -1;
}
// Add the graph to the session
tensorflow::Status status_create = session->Create(graphdef);
if (!status_create.ok()) {
std::cout << "ERROR: Creating graph in session failed..." << status_create.ToString() << std::endl;
return -1;
}
return 0;
}
/**
* @brief: Making new prediction
* @param: Session* session
* @param: FeatureAdapterBase, common interface of input feature
* @param: std::string, output_node, tensorname of output node
* @param: double, prediction values
* */
int ANNModelLoader::predict(tensorflow::Session* session, const FeatureAdapterBase& input_feature,
const std::string output_node, double* prediction) {
// The session will initialize the outputs
std::vector<tensorflow::Tensor> outputs; //shape [batch_size]
// @input: vector<pair<string, tensor> >, feed_dict
// @output_node: std::string, name of the output node op, defined in the protobuf file
tensorflow::Status status = session->Run(input_feature.input, {output_node}, {}, &outputs);
if (!status.ok()) {
std::cout << "ERROR: prediction failed..." << status.ToString() << std::endl;
return -1;
}
//Fetch output value
std::cout << "Output tensor size:" << outputs.size() << std::endl;
for (std::size_t i = 0; i < outputs.size(); i++) {
std::cout << outputs[i].DebugString();
}
std::cout << std::endl;
Tensor t = outputs[0]; // Fetch the first tensor
int ndim = t.shape().dims(); // Get the dimension of the tensor
auto tmap = t.tensor<float, 2>(); // Tensor Shape: [batch_size, target_class_num]
int output_dim = t.shape().dim_size(1); // Get the target_class_num from 1st dimension
std::vector<double> tout;
// Argmax: Get Final Prediction Label and Probability
int output_class_id = -1;
double output_prob = 0.0;
for (int j = 0; j < output_dim; j++) {
std::cout << "Class " << j << " prob:" << tmap(0, j) << "," << std::endl;
if (tmap(0, j) >= output_prob) {
output_class_id = j;
output_prob = tmap(0, j);
}
}
// Log
std::cout << "Final class id: " << output_class_id << std::endl;
std::cout << "Final value is: " << output_prob << std::endl;
(*prediction) = output_prob; // Assign the probability to prediction
return 0;
}
}
ann_model_loader.cpp 文件
#include <iostream>
#include "tensorflow/core/public/session.h"
#include "tensorflow/core/platform/env.h"
#include "ann_model_loader.h"
using namespace tensorflow;
int main(int argc, char* argv[]) {
if (argc != 2) {
std::cout << "WARNING: Input Args missing" << std::endl;
return 0;
}
std::string model_path = argv[1]; // Model_path *.pb file
// TensorName pre-defined in python file, Need to extract values from tensors
std::string input_tensor_name = "inputs";
std::string output_tensor_name = "outputs";
// Create New Session
Session* session;
Status status = NewSession(SessionOptions(), &session);
if (!status.ok()) {
std::cout << status.ToString() << "\n";
return 0;
}
// Create prediction demo
tf_model::ANNModelLoader model; //Create demo for prediction
if (0 != model.load(session, model_path)) {
std::cout << "Error: Model Loading failed..." << std::endl;
return 0;
}
// Define Input tensor and Feature Adapter
// Demo example: [1.0, 1.0, 1.0, 1.0, 1.0] for Iris Example, including bias
int ndim = 5;
std::vector<double> input;
for (int i = 0; i < ndim; i++) {
input.push_back(1.0);
}
// New Feature Adapter to convert vector to tensors dictionary
tf_model::ANNFeatureAdapter input_feat;
input_feat.assign(input_tensor_name, &input); //Assign vec<double> to tensor
// Make New Prediction
double prediction = 0.0;
if (0 != model.predict(session, input_feat, output_tensor_name, &prediction)) {
std::cout << "WARNING: Prediction failed..." << std::endl;
}
std::cout << "Output Prediction Value:" << prediction << std::endl;
return 0;
}
将这四个文件放在同一个路径下，然后还需要添加一个Cmake的txt文件：
cmake_minimum_required(VERSION 3.13)
project(cpptensorflow)
set(CMAKE_CXX_STANDARD 11)
link_directories(/Users/wanglinqing/tensorflow/bazel-bin/tensorflow)
include_directories(
/Users/wanglinqing/tensorflow
/usr/local/include
/usr/local/lib
/Users/wanglinqing/tensorflow/bazel-genfiles
/Users/wanglinqing/tensorflow/bazel-bin/tensorflow
/usr/local/Cellar/eigen/3.3.7/include/eigen3
/usr/local/include/tf/tensorflow/contrib/makefile/downloads/absl
)
add_executable(cpptensorflow main.cpp ann_model_loader.h model_loader_base.h ann_model_loader.cpp)
target_link_libraries(cpptensorflow tensorflow_cc tensorflow_framework)
这里注意cmake_minimum_required(VERSION 3.13)要和自己系统的cmake最低版本相符合。
然后在当前目录下建立一个build的空文件夹：
mkdir build
cd build
cmake ..
make
生成cpptensorflow执行文件，后接保存的模型pb文件路径：
./cpptensorflow /Users/zhoumeixu/Documents/python/credit-nlp-ner/model/liner.pb
Final value is: 14.9985
Output Prediction Value:14.9985
结果比较
在相同模型情况下：
Python运行.pb模型cpu占用时间：
129.483ms 128.369ms
128.377ms 125.818ms
131.192ms 133.627ms
128.657ms 129.095ms
128.841ms 131.273ms
C++运行.pb模型cpu占用时间：
72ms 66ms
72ms 71ms
72ms 67ms
72ms 70ms
72ms 67ms