Styjun

I am trying to build an image captioning RNN by following the same logic in assignment3 from Standford's CS231n class (http://cs231n.github.io/assignments2018/assignment3/). I had previously completed the assignment3 which used numpy to build the RNN. Right now, I want to build the same RNN but using Tensorflow this time, and the following is my code. However, the loss I get during training does not decrease for some reason. So, I think there must be some logical and setting issue within my code.

There are two inputs for this RNN during training, features and captions. The features is the output from a CNN, and captions is a matrix descibing words in each time point. Features is projected to become the initial output h0 by matrix multiplication. h0 with a zero matrix will form an initial state for the lstm. Captions (captions_in) will transform to a word embedding matrix which contains inputs for all the time point for the RNN. the outputs from the lstm will project to a scores matrix which is used for predicting the captions of the images. The scores matrix along with the captions_out matrix are used to evaluate the loss.

import tensorflow as tf
import tensorflow.contrib.eager as tfe
import numpy as np

tfe.enable_eager_execution()

class CaptioningRNN(object):
 def __init__(self, word_to_idx, input_dim=512, wordvec_dim=128, hidden_dim=128):
 self.word_to_idx = word_to_idx
 self.idx_to_word = i: w for w, i in word_to_idx.items()
 self.hidden_dim = hidden_dim

 # initialize weights
 self.W_proj = tfe.Variable(np.random.randn(input_dim, hidden_dim)/np.sqrt(input_dim), dtype=tf.float32)
 self.b_proj = tfe.Variable(np.zeros(hidden_dim), dtype=tf.float32)
 self.W_embed = tfe.Variable(np.random.randn(len(word_to_idx), wordvec_dim), dtype=tf.float32)
 self.W_vocab = tfe.Variable(np.random.randn(hidden_dim, len(word_to_idx))/np.square(hidden_dim), dtype=tf.float32)
 self.b_vocab = tfe.Variable(np.zeros(len(word_to_idx)), dtype=tf.float32)

 # initialize lstm cell
 self.encoder_cell = tf.nn.rnn_cell.BasicLSTMCell(hidden_dim)


 def forward(self, features, captions):
 outputs = []

 h0 = tf.matmul(features, self.W_proj) + self.b_proj
 x = tf.nn.embedding_lookup(self.W_embed, captions)
 timestep_x = tf.unstack(x, axis=1)

 c = tfe.Variable(np.zeros((features.shape[0], self.hidden_dim)), dtype=tf.float32)
 state = tf.nn.rnn_cell.LSTMStateTuple(c=c, h=h0)

 for input_steps in timestep_x:
 output, state = self.encoder_cell(input_steps, state)

 outputs.append(output)

 outputs = tf.stack(outputs, axis=1)

 reshaped_outputs = tf.reshape(outputs, [outputs.shape[0] * outputs.shape[1], outputs.shape[2]])
 reshaped_scores = tf.matmul(reshaped_outputs, self.W_vocab)
 scores = tf.reshape(reshaped_scores, [outputs.shape[0], outputs.shape[1], self.b_vocab.shape[0]]) + self.b_vocab

 return scores


 def cost(self, scores, labels):
 scores_flat = tf.reshape(scores, [scores.shape[0]*scores.shape[1], scores.shape[2]])
 labels_flat = tf.reshape(labels, [labels.shape[0] * labels.shape[1], 1])

 cost = tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels_flat, logits=scores_flat)
 return tf.reduce_mean(cost)


def get_loss(model, features, captions):
 scores = model.forward(features, captions[:, :-1])
 cost = model.cost(scores, captions[:, 1:])
 return cost


N = 2 # batch_size
T = 3 # timesteps
D = 4 # input_dim
W = 5 # wordvec_dim
H = 10 # hidden_dim
word_to_idx = '<NULL>': 0, 'cat': 2, 'dog': 3
vocab_size = len(word_to_idx)

features = np.random.randn(N, D).astype(np.float32)
captions = np.random.randint(vocab_size, size=(N, T))

model = CaptioningRNN(word_to_idx, input_dim=D, wordvec_dim=W, hidden_dim=H)
# get_loss(model, features, captions)

optimizer = tf.train.AdamOptimizer(5e-3)

for i in range(5000):
 optimizer.minimize(lambda: get_loss(model, features, captions))

 if i % 500 == 0:
 ls = get_loss(model, features, captions)
 print("loss: ".format(ls))

loss: 2.4719245433807373

loss: 2.4723522663116455

loss: 2.472407579421997

loss: 2.4723868370056152

loss: 2.472378969192505

loss: 2.4724044799804688

loss: 2.472418785095215

loss: 2.472426414489746

loss: 2.472430944442749

loss: 2.4724338054656982