Styjun

Following the instructions on tf custom estimator

I have created a cnn estimator and tried to train it. While training, i initialized tensorboard and was hoping to see some visualizations about training steps. However, tensorboard only showed the graph of my custom estimator but none of the scalar values i have defined.

Here's roughly what I have in code

def model_fn(features, labels, mode, params=None):
 tf.logging.set_verbosity(tf.logging.INFO)
 n_classes = params['n_classes']
 base_learning_rate = params['learning_rate']
 decay_rate = params['decay_rate']
 embedding_dim = params['embedding_dim']

 x = VGG_block1(features, (3, 3), 64, name='block1_1')
 x = VGG_block1(x, (3, 3), 128, name='block1_2')
 x = VGG_block1(x, (3, 3), 256, name='block1_3', regularizer=tf.contrib.layers.l1_regularizer(.1))
 x = VGG_block2(x, (3, 3), 512, name='block2_4')
 x = VGG_block2(x, (3, 3), 1024, name='block2_5')
 x = conv2d(x, 512, (5, 5), padding='valid', normalizer_fn=batch_norm, activation_fn=tf.nn.leaky_relu,
 weights_initializer=he_uniform())
 x = flatten(x)
 embedding = fully_connected(x, embedding_dim)
 logits = fully_connected(embedding, n_classes)

 # make predictions
 predictions = 
 'classes': tf.argmax(logits, axis=1, name='classes'),
 'probabilities': tf.nn.softmax(logits, name='softmax'),
 'embeddings':embedding
 

 # if we are in prediction mode
 if mode == tf.estimator.ModeKeys.PREDICT:
 return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)

 # otherwise define losses for training
 c_loss, center = center_loss(embedding, labels, .9, n_classes)
 xent_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=logits))
 total_loss = xent_loss + 0.5 * c_loss

 # evaluation methods
 accuracy, update_op = tf.metrics.accuracy(labels=labels, predictions=predictions['classes'], name='accuracy')
 batch_acc = tf.reduce_mean(tf.cast(tf.equal(tf.cast(labels, tf.int64), predictions['classes']), tf.float32))
 tf.summary.scalar('batch_acc', batch_acc)
 tf.summary.scalar('streaming_acc', update_op)
 tf.summary.scalar('total_loss', total_loss)
 tf.summary.scalar('center_loss', c_loss)
 tf.summary.scalar('xent_loss', xent_loss)

 # training mode
 if mode == tf.estimator.ModeKeys.TRAIN:
 update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
 global_step = tf.Variable(0, trainable=False)
 global_step_op = tf.assign(global_step, global_step + 1)
 learning_rate = tf.train.exponential_decay(base_learning_rate, global_step, 8000, decay_rate, staircase=True)
 optimizer = tf.train.AdamOptimizer(learning_rate)
 with tf.control_dependencies(update_ops+[global_step_op]):
 objective = optimizer.minimize(total_loss)

 return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, train_op=objective)

 eval_metric_ops = 
 'accuracy': (accuracy, update_op)
 
 return tf.estimator.EstimatorSpec(mode=mode, loss=total_loss, eval_metric_ops=eval_metric_ops)

X_train, X_test, y_train, y_test = load_data()

epochs = 10
batch_size = 64
n_classes = len(classes)

model_params = 'n_classes':n_classes,
 'learning_rate':0.0001,
 'decay_rate':0.5,
 'embedding_dim':128
model_dir = 'output'
face_classifier = tf.estimator.Estimator(model_fn=model_fn, params=model_params, model_dir=model_dir)

My Tensorflow version is 1.12.0

Edit
Forgot to mention I was using eager execution for this exercise, for unknown reasons that was the cause of this bug

as was mentioned in the edit, disabling eager execution solved the problem