1. convert keras model .h5 to tensorflow serving model
import tensorflow as tf
from keras import backend as K
from keras.models import Sequential, Model
from os.path import isfile
from keras.layers import Dense, Dropout, Activation
from keras.layers import Embedding
from keras.layers import Conv1D, GlobalMaxPooling1D
import os
max_features = 449590
maxlen = 300
batch_size = 32
embedding_dims = 50
filters = 250
kernel_size = 3
hidden_dims = 250
def build_model():
model = Sequential()
model.add(Embedding(max_features,
embedding_dims,
input_length=maxlen))
model.add(Dropout(0.5))
model.add(Conv1D(filters,
kernel_size,
padding='valid',
activation='relu',
strides=1))
model.add(GlobalMaxPooling1D())
model.add(Dense(hidden_dims))
model.add(Dropout(0.5))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def save_model_to_serving(model, export_version, export_path='prod_models'):
print(model.input, model.output)
signature = tf.saved_model.signature_def_utils.predict_signature_def(
inputs={'inputs': model.input}, outputs={'outputs': model.output})
export_path = os.path.join(
tf.compat.as_bytes(export_path),
tf.compat.as_bytes(str(export_version)))
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess=K.get_session(),
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map={
'news_classification': signature,
},
legacy_init_op=legacy_init_op)
builder.save()
if __name__ == '__main__':
model = build_model()
model.summary()
checkpoint_filepath = 'model.h5'
if (isfile(checkpoint_filepath)):
print('Checkpoint file detected. Loading weights.')
model.load_weights(checkpoint_filepath) # 加载模型
model.summary()
export_path = "tf_model"
# model, version, model save path
save_model_to_serving(model, "1", export_path)
else:
print('No checkpoint file detected. Starting from scratch.')
'''
output dir list:
tf_model
--1
||--saved_model.pb
||--variables
||||----variables.data-00000-of-00001
||||----variables.index
'''2. tensorflow grpc client
import sys
sys.path.insert(0, "./")
from tensorflow_serving_client.protos import predict_pb2, prediction_service_pb2
import cv2
import grpc
from grpc.beta import implementations
import tensorflow as tf
from tensorflow.python.framework import dtypes
import time
import numpy as np
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
if __name__ == '__main__':
# x = [346,274,242,454,345,705,515,242,6166,538,182,450,252,261,2637,3343,15,45,4,2,4,188,1257,1086,5200,526,15,4090,18,242,43,18,1671,117,3265,16179,105,242,18,6,105,14877,531,1439,3173,12438,450,110,15,108,1841,1365,3705,143,1013,208603,143,601,35212,3705,1,48,5686,252,601,814,161,925,182,35,55,515,925,479,325,1841,3705,378,601,801,4407,12878,221,1841,3705,1725,1456,37426,515,4,125,267,4,297,45,3351,554,74,2447,74,147,3202,104,925,242,74,423,5462,479,423,925,1589,666,4,925,18,2,301,478,122,101,75617,12434,297,370,1,125,74,1,219,283,580,6,6720,486,122,1752,320,19,307,1841,148,161,58,58,1689,113,2557,925,580,824,2784,10396,2808,242,1653,925,6370,58,590,6252,35,68,263,796,273,2273,143,14926,143,16512,168,4206,2170,2559,6370,604,14,2,1301,56,122,1565,31,400,925,125,221,9723,191,9,1004,1273,1714,74,1565,125,1248,3202,6639,219,249,1565,934,222,54866,5422,861,1565,9124,5,6,249,7668,28611,205,471,8613,397,63,402,13149,1565,502,1565,1456,11384,5559,99,1092,925,1092,188,57968,191,203,658,655,108,3078,2438,1086,43,4493,33,886,107,35,7148,167,3,3,2154,243698,891,43,2154,15781,5865,43,2149,14954,21933,33,77,1092,8613,3396,108,24,25605,540,152,150,1092,1070,566,50,1092,188,1092,925,25605,1,659,3855,1092,125,297,10389,3855,630,759,24,400,3237,4,13104,379,283]
x=[4859,10307,574,1402,4556,33674,4337,698,14731,10032,117,1458,323,1513,570,3693,570,290,1006,14731,589,17116,9506,847,103,419,378,847,103,419,1785,1048,7,847,103,3875,29,199,33135,229,2614,2959,229,36,229,847,103,419,8045,11906,17143,1048,3875,238,1311,191,323,3630,7,8755,39082,8071,12242,33740,16988,219373,7,8755,39082,8071,12242,33740,16988,219373,29,66,1648,68126,102865,1068,131912,65,847,103,7,133612,1163,662,26926,456,452,799,163130,39082,39082,24685,39082,1758,256,151688,880,8542,8759,166918,914,57470,21967,2271,39082,1311,33683,57470,2614,91416,191644,32577,57470,2614,91416,191644,32577,57470,880,516,39082,3716,380,600,490,72,14603,3716,39082,2990,7,5660,641,12362,68126,591,7346,3716,39082,3873,667,57470,865,1940,8470,57470,2150,16905,58550,131280,233409,57470,2150,16905,58550,131280,233409,8470,2931,65024,1285,39082,8071,57470,39082,256,1875,58716,847,103,4534,990,3941,24346,229,3492,647,1013,3790,7,133333,68126,131912,608,5535,1163,7,550,238,2869,57470,4475,1013,229,57470,2150,29,131912,23019,66378,175423,328,1673,1068,16144,39082,654,2133,6033,55192,175,39082,1758,600,119,2387,64717,28855,111,54589,14758,431156,65,503,1427,9949,6033,55192,16038,55192,490,855,852,23622,57470,23012,3841,42932,151901,57470,23012,3841,42932,151901,280652,191,7,379503,379142,39082,20755,1040,13137,5434,3917,7,1855,68126,10193,7,38536,238,1403,311,13032,2028,20755,12493,1040,13137,5434,3917,7,1855,7,5536,2248,2475,258,20755,323,29,199,806,29,199,13655,5522,20755,8179,238,2883,585,412,20755,32362,
4859,10307,574,1402,4556,33674,4337,698,14731,10032,117,1458,323,1513,570,3693,570,290,1006,14731,589,17116,9506,847,103,419,378,847,103,419,1785,1048,7,847,103,3875,29,199,33135,229,2614,2959,229,36,229,847,103,419,8045,11906,17143,1048,3875,238,1311,191,323,3630,7,8755,39082,8071,12242,33740,16988,219373,7,8755,39082,8071,12242,33740,16988,219373,29,66,1648,68126,102865,1068,131912,65,847,103,7,133612,1163,662,26926,456,452,799,163130,39082,39082,24685,39082,1758,256,151688,880,8542,8759,166918,914,57470,21967,2271,39082,1311,33683,57470,2614,91416,191644,32577,57470,2614,91416,191644,32577,57470,880,516,39082,3716,380,600,490,72,14603,3716,39082,2990,7,5660,641,12362,68126,591,7346,3716,39082,3873,667,57470,865,1940,8470,57470,2150,16905,58550,131280,233409,57470,2150,16905,58550,131280,233409,8470,2931,65024,1285,39082,8071,57470,39082,256,1875,58716,847,103,4534,990,3941,24346,229,3492,647,1013,3790,7,133333,68126,131912,608,5535,1163,7,550,238,2869,57470,4475,1013,229,57470,2150,29,131912,23019,66378,175423,328,1673,1068,16144,39082,654,2133,6033,55192,175,39082,1758,600,119,2387,64717,28855,111,54589,14758,431156,65,503,1427,9949,6033,55192,16038,55192,490,855,852,23622,57470,23012,3841,42932,151901,57470,23012,3841,42932,151901,280652,191,7,379503,379142,39082,20755,1040,13137,5434,3917,7,1855,68126,10193,7,38536,238,1403,311,13032,2028,20755,12493,1040,13137,5434,3917,7,1855,7,5536,2248,2475,258,20755,323,29,199,806,29,199,13655,5522,20755,8179,238,2883,585,412,20755,32362]
start_time = time.time()
channel = grpc.insecure_channel("172.17.0.5:8502")
# stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = 'news_classifier'
request.model_spec.signature_name='news_classification'
request.inputs["inputs"].ParseFromString(tf.contrib.util.make_tensor_proto(x, dtype=dtypes.float32, shape=[2,300]).SerializeToString())
response = stub.Predict(request, 10.0)
results = {}
print(response.outputs)notice:
a. client code : request.model_spec.name and request.model_spec.signature_name should be consistent with save mode config
deployment in docker
step 1 : fetch tensorflow serving docker image
docker pull tensorflow/serving
step 2 : run a container from docker image
(1). run in a container :
tensorflow_model_server --port=8502 --model_name="news_classifier" --model_base_path="/models/news_classifier"
(2). run on host machine:
docker run -p 8502:8502 --mount type=bind,source=tensorflow_serving_test/tf_model,target=/models/news_classifier -e MODEL_NAME=news_classifier -t tensorflow/serving
3. call predict with restful api
when call start model server with rest api , rest_api_port must be assigned
tensorflow_model_server --port=8500 --rest_api_port=8501 --model_name=news_classifier --model_base_path=/models/news_classifier
model_name is same to api url 'http://ip:8501/v1/models/{model_name}:predict'
3.1 tensor shape is wrong
curl -d '{"signature_name":"news_classification","inputs": [4859,10307,574,1402,4556,33674,4337,698,14731,10032,117,1458,323,1513,570,3693,570,290,1006,14731,589,17116,9506,847,103,419,378,847,103,419,1785,1048,7,847,103,3875,29,199,33135,229,2614,2959,229,36,229,847,103,419,8045,11906,17143,1048,3875,238,1311,191,323,3630,7,8755,39082,8071,12242,33740,16988,219373,7,8755,39082,8071,12242,33740,16988,219373,29,66,1648,68126,102865,1068,131912,65,847,103,7,133612,1163,662,26926,456,452,799,163130,39082,39082,24685,39082,1758,256,151688,880,8542,8759,166918,914,57470,21967,2271,39082,1311,33683,57470,2614,91416,191644,32577,57470,2614,91416,191644,32577,57470,880,516,39082,3716,380,600,490,72,14603,3716,39082,2990,7,5660,641,12362,68126,591,7346,3716,39082,3873,667,57470,865,1940,8470,57470,2150,16905,58550,131280,233409,57470,2150,16905,58550,131280,233409,8470,2931,65024,1285,39082,8071,57470,39082,256,1875,58716,847,103,4534,990,3941,24346,229,3492,647,1013,3790,7,133333,68126,131912,608,5535,1163,7,550,238,2869,57470,4475,1013,229,57470,2150,29,131912,23019,66378,175423,328,1673,1068,16144,39082,654,2133,6033,55192,175,39082,1758,600,119,2387,64717,28855,111,54589,14758,431156,65,503,1427,9949,6033,55192,16038,55192,490,855,852,23622,57470,23012,3841,42932,151901,57470,23012,3841,42932,151901,280652,191,7,379503,379142,39082,20755,1040,13137,5434,3917,7,1855,68126,10193,7,38536,238,1403,311,13032,2028,20755,12493,1040,13137,5434,3917,7,1855,7,5536,2248,2475,258,20755,323,29,199,806,29,199,13655,5522,20755,8179,238,2883,585,412,20755,32362]}' -X POST http://172.17.0.5:8501/v1/models/news_classifier:predict
{ "error": "input must be 4-dimensional[300,1,50]\n\t [[{{node conv1d_1/convolution/Conv2D}} = Conv2D[T=DT_FLOAT, _output_shapes=[[?,1,298,250]], data_format=\"NHWC\", dilations=[1, 1, 1, 1], padding=\"VALID\", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](conv1d_1/convolution/ExpandDims, conv1d_1/convolution/ExpandDims_1)]]" }[xiewenbo@oxpecker tensorflow_serving_test]$
input must be 4-dimensional, first demension is samples number
curl -d '{"signature_name":"news_classification","inputs": [[4859,10307,574,1402,4556,33674,4337,698,14731,10032,117,1458,323,1513,570,3693,570,290,1006,14731,589,17116,9506,847,103,419,378,847,103,419,1785,1048,7,847,103,3875,29,199,33135,229,2614,2959,229,36,229,847,103,419,8045,11906,17143,1048,3875,238,1311,191,323,3630,7,8755,39082,8071,12242,33740,16988,219373,7,8755,39082,8071,12242,33740,16988,219373,29,66,1648,68126,102865,1068,131912,65,847,103,7,133612,1163,662,26926,456,452,799,163130,39082,39082,24685,39082,1758,256,151688,880,8542,8759,166918,914,57470,21967,2271,39082,1311,33683,57470,2614,91416,191644,32577,57470,2614,91416,191644,32577,57470,880,516,39082,3716,380,600,490,72,14603,3716,39082,2990,7,5660,641,12362,68126,591,7346,3716,39082,3873,667,57470,865,1940,8470,57470,2150,16905,58550,131280,233409,57470,2150,16905,58550,131280,233409,8470,2931,65024,1285,39082,8071,57470,39082,256,1875,58716,847,103,4534,990,3941,24346,229,3492,647,1013,3790,7,133333,68126,131912,608,5535,1163,7,550,238,2869,57470,4475,1013,229,57470,2150,29,131912,23019,66378,175423,328,1673,1068,16144,39082,654,2133,6033,55192,175,39082,1758,600,119,2387,64717,28855,111,54589,14758,431156,65,503,1427,9949,6033,55192,16038,55192,490,855,852,23622,57470,23012,3841,42932,151901,57470,23012,3841,42932,151901,280652,191,7,379503,379142,39082,20755,1040,13137,5434,3917,7,1855,68126,10193,7,38536,238,1403,311,13032,2028,20755,12493,1040,13137,5434,3917,7,1855,7,5536,2248,2475,258,20755,323,29,199,806,29,199,13655,5522,20755,8179,238,2883,585,412,20755,32362]]}' -X POST http://172.17.0.5:8501/v1/models/news_classifier:predict
reference:
1. https://blog.csdn.net/u011734144/article/details/82107610?utm_source=oschina-app
2. https://blog.csdn.net/shin627077/article/details/78592729
3. https://www.jianshu.com/p/2fffd0e332bc
4. https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/serving_basic.md
5. https://medium.com/@yuu.ishikawa/introduction-to-restful-api-with-tensorflow-serving-9c60969b5b95
6. https://medium.com/@yuu.ishikawa/how-to-show-signatures-of-tensorflow-saved-model-5ac56cf1960f
7. https://www.tensorflow.org/serving/api_rest
两种部署运行方式：
1. bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=8500 --model_name=test --model_base_path=/models/test/
2.配置文件
model_config_list: {
config: {
name: "mymodel",
base_path: "/some/filesystem/path",
model_platform: "tensorflow",
model_version_policy: {
specific: {
versions: 101,
versions: 202
}
}
},
config: {
name: "mymodel2",
base_path: "/some/filesystem/path2",
model_platform: "tensorflow",
model_version_policy: {
latest: {
num_versions: N
}
}
},
}
other:
1. Input array should use ' , ' as the delemiter, while whitespace will cause json parse error
Wrong Format
66 67 487 159 3 135 1 20 10 138 227 36 27 2 82 318 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
Right Format
361,54,7,1301,238,1,334,83,1171,5138,1217,1127,806,4,319,913,455,1129,1,231,90,122,364,1,79,75,62,526,968,464,101,59,320,231,409,468,214,51,1,243,53,695,499,420,1217,1127,2,369,260,1,82,59,33,48,455,1129,10,3980,1129,2,43,478,478,33,350,51,34,1,361,54,455,1129,10,168,408,7,226,664,53,224,446,297,464,101,1217,1127,1,3,41,1336,53,140,5,14,51,7,226,46,226,457,838,908,2,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0