TensorFlow - save model

There are some examples to show how to save trained models 

example 1:

this code can save the model which achieves max acc; 
but it cannot train model based on the pre-trained model; 
Besides, it evaluates the performances only in batch level rather than in epoch level.

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets("D:/vsCode_tensorflow/PKU_TF/PKU_TF_shuzi/data2", one_hot=False)

x = tf.placeholder(tf.float32, [None, 784])
y_=tf.placeholder(tf.int32,[None,])

dense1 = tf.layers.dense(inputs=x,
                      units=1024,
                      activation=tf.nn.relu,
                      kernel_initializer=tf.truncated_normal_initializer(stddev=0.01),
                      kernel_regularizer=tf.nn.l2_loss)
dense2= tf.layers.dense(inputs=dense1,
                      units=512,
                      activation=tf.nn.relu,
                      kernel_initializer=tf.truncated_normal_initializer(stddev=0.01),
                      kernel_regularizer=tf.nn.l2_loss)
logits= tf.layers.dense(inputs=dense2,
                        units=10,
                        activation=None,
                        kernel_initializer=tf.truncated_normal_initializer(stddev=0.01),
                        kernel_regularizer=tf.nn.l2_loss)

loss=tf.losses.sparse_softmax_cross_entropy(labels=y_,logits=logits)
train_op=tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)
correct_prediction = tf.equal(tf.cast(tf.argmax(logits,1),tf.int32), y_)
acc= tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

sess=tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

is_train = False
saver = tf.train.Saver(max_to_keep=5)
# this code can save the model which achieve max acc

if is_train:
    max_acc=0
    f=open('D:/DL/TF21_12/acc.txt', 'w')
    for i in range(100):
        batch_xs, batch_ys = mnist.train.next_batch(100)
        sess.run(train_op, feed_dict={x: batch_xs, y_: batch_ys})
        val_loss, val_acc=sess.run([loss, acc], feed_dict={x: mnist.test.images, y_: mnist.test.labels})
        print('epoch:%d, val_loss:%f, val_acc:%f' % (i, val_loss, val_acc))
        f.write(str(i+1)+', val_acc: '+str(val_acc)+'\n')
        if val_acc>max_acc:
            max_acc=val_acc
            saver.save(sess,'D:/DL/TF21_12/all_ckpt2/mnist.ckpt', global_step=i+1)
    f.close()
else:
    model_file=tf.train.latest_checkpoint('D:/DL/TF21_12/all_ckpt2/')
    saver.restore(sess, model_file)
    val_loss,val_acc = sess.run([loss,acc], feed_dict={x: mnist.test.images, y_: mnist.test.labels})
    print('val_loss:%f, val_acc:%f' % (val_loss, val_acc))
sess.close()

From 

# tensorflow 1.0 学习：模型的保存与恢复(Saver) - denny402 - 博客园
# https://www.cnblogs.com/denny402/p/6940134.html

example 2:

this code can ensure that the saved model is the one which gets the best performances in the training data; 
however, performance should be evaluated on validation data; 
By the way, the code     ckpt = tf.train.get_checkpoint_state(model_path)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("OK, this is the restored model")can only judge whether the path exists; it cannot select the best one in the diretory if there are more than one model.

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

mnist = input_data.read_data_sets("D:/vsCode_tensorflow/PKU_TF/PKU_TF_shuzi/data2", one_hot=True)
# Parameters
learning_rate = 0.001
batch_size = 100
display_step = 1
model_path = "D:/DL/TF21_12/all_ckpt3/"

# Network Parameters
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
model_global_step = 0

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])

# Store layers weight & bias
weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}


# Create model
def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.relu(layer_1)
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.relu(layer_2)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer


# Construct model
pred = multilayer_perceptron(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

# Initializing the variables
init = tf.global_variables_initializer()

# 'Saver' op to save and restore all the variables
saver = tf.train.Saver(max_to_keep=1)

is_train = False

# 这个例子先check有没有训练好的model，有则拿过来继续训练
with tf.Session() as sess:
    # Initialize variables
    sess.run(init)

    ckpt = tf.train.get_checkpoint_state(model_path)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("OK, this is the restored model")

    if is_train:
        # Training cycle, 5 epoch
        for epoch in range(5):
            model_global_step = model_global_step+1
            avg_cost = 0
            avg_acc = 0
            max_acc = 0
            total_batch = int(mnist.train.num_examples/batch_size)
            # Loop over all batches
            for i in range(total_batch):
                batch_x, batch_y = mnist.train.next_batch(batch_size)
                _, loss, acc = sess.run([optimizer, cost, accuracy], feed_dict={x: batch_x, y: batch_y})
                # Compute average loss of each epoch
                avg_cost += loss / total_batch
                avg_acc += acc / total_batch

            # Display logs per epoch step
            if epoch % display_step == 0:
                print("Epoch:", '%04d' % (epoch+1),
                  "cost=", "{:.9f}".format(avg_cost), "accuracy=", "{:.9f}".format(avg_acc))

            if avg_acc > max_acc:
                max_acc=avg_acc
                saver.save(sess, model_path+'model.ckpt', global_step=model_global_step+1)

        print("First Optimization Finished!")

    else:
        # Test model
        model_file = tf.train.latest_checkpoint(model_path)
        saver.restore(sess, model_file)
        print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))

example 3：

Tensorflow - 将序列处理成embedding - 方法1 - keras调包