Long Nguyen :
While trying to loading my trained Keras model for further inference with
self.model = tf.keras.models.load_model(filepath=weight_url, custom_objects={
'dice_coef': dice_coef,
'iou_coef': iou_coef,
'bce_dice_coef': bce_dice_coef,
})
I get the following error. Those three functions are the same I used to train my model, the weight URL is correct. The error message doesn't tell me anything.
file "/home/long/Desktop/bachelor-thesis/unet/pipeline.py", line 344, in <module>
binary.inference_blood_cell_unet()
File "/home/long/Desktop/bachelor-thesis/unet/pipeline.py", line 305, in inference_blood_cell_unet
self.blood_cell_unet = UNet(weight_url=self.blood_cell_final_name, class_weights=self.blut_koerperchen_classweights, num_classes=2)
File "/home/long/Desktop/bachelor-thesis/unet/model.py", line 39, in __init__
'bce_dice_coef': bce_dice_coef,
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/save.py", line 146, in load_model
return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/saving/hdf5_format.py", line 193, in load_model_from_hdf5
model._make_train_function()
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 2116, in _make_train_function
params=self._collected_trainable_weights, loss=self.total_loss)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py", line 500, in get_updates
grads = self.get_gradients(loss, params)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py", line 391, in get_gradients
grads = gradients.gradients(loss, params)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_impl.py", line 158, in gradients
unconnected_gradients)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py", line 550, in _GradientsHelper
gradient_uid)
File "/home/long/Desktop/bachelor-thesis/venv/lib/python3.7/site-packages/tensorflow_core/python/ops/gradients_util.py", line 166, in _DefaultGradYs
with _maybe_colocate_with(y.op, gradient_uid, colocate_gradients_with_ops):
AttributeError: 'NoneType' object has no attribute 'op'
Following is my code how the model is built:
import tensorflow as tf
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, LeakyReLU, Dropout, MaxPooling2D, Conv2DTranspose
from tensorflow.keras.layers import concatenate
from tensorflow.keras.optimizers import Adam
from unet.metrics import dice_coef, iou_coef, cce_dice_coef, weighted_loss, bce_dice_coef
fil_coef = 4
class UNet(UNetBase, DataHandlingBase):
def __init__(self,
class_weights,
weight_url=None,
width=256,
height=256,
channel=3,
learning_rate=0.0005,
num_classes=2,
alpha=0.01,
dropout_rate=0.25):
super().__init__(weight_url, width, height, channel, learning_rate, num_classes, alpha, dropout_rate)
self.class_weights = class_weights
if self.num_classes > 2:
self.loss_function = weighted_loss(cce_dice_coef, weights_list=self.class_weights)
elif self.num_classes == 2:
self.loss_function = bce_dice_coef
else:
raise Exception("Invalid num class: ", self.num_classes)
if weight_url:
self.model = tf.keras.models.load_model(filepath=weight_url, custom_objects={
'dice_coef': dice_coef,
'iou_coef': iou_coef,
'bce_dice_coef': bce_dice_coef,
'loss_function': self.loss_function
})
else:
model_input = Input((self.height, self.width, self.channel))
c1 = Conv2D(filters=16 * fil_coef, kernel_size=(3, 3), padding='same')(model_input)
c1 = BatchNormalization()(c1)
c1 = LeakyReLU(self.alpha)(c1)
c1 = Dropout(self.dropout_rate)(c1)
c1 = Conv2D(filters=16 * fil_coef, kernel_size=(3, 3), padding='same')(c1)
...
c9 = BatchNormalization()(c9)
c9 = LeakyReLU(self.alpha)(c9)
c9 = Dropout(self.dropout_rate)(c9)
if self.num_classes == 2:
output = Conv2D(1, kernel_size=(1, 1), activation='sigmoid')(c9)
else:
output = Conv2D(self.num_classes, kernel_size=(1, 1), activation='softmax')(c9)
self.model = tf.keras.Model(inputs=[model_input], outputs=[output])
self.reinitialize()
def reinitialize(self):
if self.num_classes > 2:
self.model.compile(optimizer=Adam(lr=self.learning_rate),
loss=self.loss_function,
metrics=[dice_coef, iou_coef])
else:
self.model.compile(optimizer=Adam(lr=self.learning_rate),
loss=self.loss_function,
metrics=[dice_coef, iou_coef],
class_weight=self.class_weights)
def fit(self, X, Y, x=None, y=None,
batch_size=params["batch_size"],
epochs=params["epochs"],
validation_split=params["validation_split"],
checkpoint_path="temp/models/model-{epoch:02d}-{val_loss:.2f}.h5",
patience=params["patience"],
min_delta=params["min_delta"]):
checkpoint = ModelCheckpoint(checkpoint_path,
verbose=0,
save_best_only=False,
monitor='val_loss',
mode='auto')
if x is None and y is None:
history = self.model.fit(X, Y,
validation_split=validation_split,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
callbacks=[checkpoint])
return history
elif x is not None and y is not None:
history = self.model.fit(X, Y,
verbose=0,
steps_per_epoch=X.shape[0] // batch_size,
validation_data=[x, y],
validation_steps=x.shape[0] // batch_size,
batch_size=batch_size,
epochs=epochs,
shuffle=True)
return history
def save_weights(self, url):
self.model.save(url)
def predict(self, X):
return self.model.predict(X)
def summary(self):
log.info(f"Learning rate {self.learning_rate} Alpha {self.alpha} Dropout Rate {self.dropout_rate}")
self.model.summary()
And here are how my loss functions coded:
import tensorflow as tf
import tensorflow.keras.backend as K
def weighted_loss(original_loss_function, weights_list):
"""
Author: https://stackoverflow.com/questions/51793737/custom-loss-function-for-u-net-in-keras-using-class-weights-class-weight-not
"""
def loss_function(true, pred):
axis = -1 # if channels last
# axis= 1 #if channels first
# argmax returns the index of the element with the greatest value
# done in the class axis, it returns the class index
class_selectors = tf.cast(K.argmax(true, axis=axis), tf.int32)
# if your loss is sparse, use only true as classSelectors
# considering weights are ordered by class, for each class
# true(1) if the class index is equal to the weight index
class_selectors = [K.equal(i, class_selectors) for i in range(len(weights_list))]
# casting boolean to float for calculations
# each tensor in the list contains 1 where ground true class is equal to its index
# if you sum all these, you will get a tensor full of ones.
class_selectors = [K.cast(x, K.floatx()) for x in class_selectors]
# for each of the selections above, multiply their respective weight
weights = [sel * w for sel, w in zip(class_selectors, weights_list)]
# sums all the selections
# result is a tensor with the respective weight for each element in predictions
weight_multiplier = weights[0]
for i in range(1, len(weights)):
weight_multiplier = weight_multiplier + weights[i]
# make sure your originalLossFunc only collapses the class axis
# you need the other axes intact to multiply the weights tensor
loss = original_loss_function(true, pred)
loss = loss * weight_multiplier
return loss
return loss_function
@tf.function
def cce_iou_coef(y_true, y_pred, smooth=1, cat_weight=1, iou_weight=1):
return cat_weight * K.categorical_crossentropy(y_true, y_pred) + iou_weight * log_iou_coef(y_true, y_pred, smooth)
@tf.function
def cce_dice_coef(y_true, y_pred, smooth=1, cat_weight=1, dice_weight=1):
return cat_weight * K.categorical_crossentropy(y_true, y_pred) + dice_weight * log_dice_coef(y_true, y_pred, smooth)
@tf.function
def bce_iou_coef(y_true, y_pred, smooth=1, cat_weight=1, iou_weight=1):
return cat_weight * K.binary_crossentropy(y_true, y_pred) + iou_weight * log_iou_coef(y_true, y_pred, smooth)
@tf.function
def bce_dice_coef(y_true, y_pred, smooth=1, cat_weight=1, dice_weight=1):
return cat_weight * K.binary_crossentropy(y_true, y_pred) + dice_weight * log_dice_coef(y_true, y_pred, smooth)
@tf.function
def log_iou_coef(y_true, y_pred, smooth=1):
return - K.log(iou_coef(y_true, y_pred, smooth))
@tf.function
def log_dice_coef(y_true, y_pred, smooth=1):
return -K.log(dice_coef(y_true, y_pred, smooth))
@tf.function
def iou_coef(y_true, y_pred, smooth=1):
intersection = K.sum(K.abs(y_true * y_pred), axis=-1)
union = K.sum(y_true, axis=-1) + K.sum(y_pred, axis=-1) - intersection
iou = K.mean((intersection + smooth) / (union + smooth), axis=0)
return iou
@tf.function
def dice_coef(y_true, y_pred, smooth=1):
intersection = K.sum(y_true * y_pred, axis=-1)
union = K.sum(y_true, axis=-1) + K.sum(y_pred, axis=-1)
dice = K.mean((2. * intersection + smooth) / (union + smooth), axis=0)
return dice
The error happens after I trained a binary model, saved it and loads it again for inference. As you can see the loss functions imported for inference are the same for the training phase. So I don't really understand why I get the error. And I don't understand what the error means, either.
SajanGohil :
You can try with compile = False parameter in your load_model call. This will remove any metadata related to optimizers and loss function and since you have a reinitialize function and if you don't need to train it from where you stopped last time, that won't be a problem I guess.
As you said it works with compile = False, I think the problem might be in your custom functions for loss/optimizer etc, I can't pin point what exactly is the problem, you can try tf. instead of tf.keras.backend. if you want.