损失函数
tf.python.ops.nn_ops.sparse_softmax_cross_rntropy_with_logits(logits, labels, name=None)
def sparse_softmax_cross_entropy_with_logits(logits, labels, name=None):
#logits是最后一层的z（输入）
#Each entry in `labels` must be an index in `[0, num_classes)` lables是类别的索引
#输出：loss [batch_size]tf.python.ops.nn_ops.softmax_cross_entropy_with_logits_v2(logits, targets, dim=-1, name=None)
def softmax_cross_entropy_with_logits(logits, targets, dim=-1, name=None):
#`logits` and `labels` must have the same shape `[batch_size, num_classes]` labels是类别的概率分布
#return loss:[batch_size], 里面保存是batch中每个样本的cross entropytf.nn.sigmoid_cross_entropy_with_logits(logits, targets, name=None)
def sigmoid_cross_entropy_with_logits(logits, targets, name=None):
#logits:[batch_size, num_classes],targets:[batch_size, size].logits作为用最后一层的输入就好，不需要进行sigmoid运算，函数内部进行了sigmoid操作。
#输出loss [batch_size, num_classes]
For brevity, let x = logits, z = labels. The logistic loss is
z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))tf.nn.nce_loss(nce_weights,nce_biases,embed,train_labels, num_sampled, vocabulary_size)
def nce_loss(nce_weights, nce_biases, embed, train_labels, num_sampled, vocabulary_size):
#word2vec中用到了这个函数
#weights: A `Tensor` of shape `[num_classes, dim]`, or a list of `Tensor`
# objects whose concatenation along dimension 0 has shape
# [num_classes, dim]. The (possibly-partitioned) class embeddings.
#biases: A `Tensor` of shape `[num_classes]`. The class biases.
#inputs: A `Tensor` of shape `[batch_size, dim]`. The forward
# activations of the input network.
#labels: A `Tensor` of type `int64` and shape `[batch_size,
# num_true]`. The target classes.
#num_sampled: An `int`. The number of classes to randomly sample per batch.
#num_classes: An `int`. The number of possible classes.
#num_true: An `int`. The number of target classes per training example.tf.nn.sequence_loss_by_example(logits, targets, weights,average_across_timesteps=True,softmax_loss_function=None, name=None)
def sequence_loss_by_example(logits, targets, weights,
average_across_timesteps=True,
softmax_loss_function=None, name=None):
#logits: List of 2D Tensors of shape [batch_size x num_decoder_symbols].
#targets: List of 1D batch-sized int32 Tensors of the same length as logits.
#weights: List of 1D batch-sized float-Tensors of the same length as logits.
#return:log_pers 形状是 [batch_size].
for logit, target, weight in zip(logits, targets, weights):
if softmax_loss_function is None:
# TODO(irving,ebrevdo): This reshape is needed because
# sequence_loss_by_example is called with scalars sometimes, which
# violates our general scalar strictness policy.
target = array_ops.reshape(target, [-1])
crossent = nn_ops.sparse_softmax_cross_entropy_with_logits(
logit, target)
else:
crossent = softmax_loss_function(logit, target)
log_perp_list.append(crossent * weight)
log_perps = math_ops.add_n(log_perp_list)
if average_across_timesteps:
total_size = math_ops.add_n(weights)
total_size += 1e-12 # Just to avoid division by 0 for all-0 weights.
log_perps /= total_size
return log_perps
## 关于weights：形状应该是[T, batch_size] ,如果input包含填充的数据，对应的weights置0，其余置1。这样就可以保证，填充的数据不会进行梯度下降。激活函数
tf.nn.relu()
tf.nn.sigmoid()
tf.nn.tanh()
tf.nn.elu()
tf.nn.bias_add()
tf.nn.crelu()
tf.nn.relu6()
tf.nn.softplus()
tf.nn.softsign()
tf.nn.dropout()
tf.nn.relu_layer(x, weights, biases,name=None)
def relu_layer(x, weights, biases, name=None):
"""Computes Relu(x * weight + biases).
Args:
x: a 2D tensor. Dimensions typically: batch, in_units
weights: a 2D tensor. Dimensions typically: in_units, out_units
biases: a 1D tensor. Dimensions: out_units
name: A name for the operation (optional). If not specified
"nn_relu_layer" is used.
Returns:
A 2-D Tensor computing relu(matmul(x, weights) + biases).
Dimensions typically: batch, out_units.
"""