Styjun

I have implemented a paper about a CNN architecture in both Keras and Pytorch but keras implementation is much more efficient it takes 4 gb of gpu for training with 50000 samples and 10000 validation samples but pytorch one takes all the 12 gb of gpu and i cant even use a validation set !
Optimizer for both of them is sgd with momentum and same settings for both.
more info about the paper:[architecture]:https://github.com/Moeinh77/Lightweight-Deep-Convolutional-Network-for-Tiny-Object-Recognition/edit/master/train.py

pytorch code :

class SimpleCNN(torch.nn.Module):

 def __init__(self):
 super(SimpleCNN, self).__init__()

 self.conv2d_11 = torch.nn.Conv2d(3, 64, kernel_size = 3, stride = 1, padding = 1)
 self.conv2d_12 = torch.nn.Conv2d(64, 64, kernel_size = 3, stride = 1, padding = 1)

 self.conv2d_21 = torch.nn.Conv2d(64, 128, kernel_size = 3, stride = 1, padding = 1)
 self.conv2d_22 = torch.nn.Conv2d(128, 128, kernel_size = 3, stride = 1, padding = 1)

 self.conv2d_31 = torch.nn.Conv2d(128, 256, kernel_size = 3, stride = 1, padding = 1)
 self.conv2d_32 = torch.nn.Conv2d(256, 256, kernel_size = 3, stride = 1, padding = 1)
 self.conv2d_33 = torch.nn.Conv2d(256, 256, kernel_size = 3, stride = 1, padding = 1)

 self.conv2d_41 = torch.nn.Conv2d(256, 512, kernel_size = 3, stride = 1, padding = 1)
 self.conv2d_42 = torch.nn.Conv2d(512, 512, kernel_size = 3, stride = 1, padding = 1)

 self.conv2d_51 = torch.nn.Conv2d(512, 512, kernel_size = 3, stride = 1, padding = 1)

 self.Batchnorm_1=torch.nn.BatchNorm2d(64)
 self.Batchnorm_2=torch.nn.BatchNorm2d(128)
 self.Batchnorm_3=torch.nn.BatchNorm2d(256)
 self.Batchnorm_4=torch.nn.BatchNorm2d(512)

 self.dropout2d_1=torch.nn.Dropout2d(p=0.3)
 self.dropout2d_2=torch.nn.Dropout2d(p=0.4)
 self.dropout2d_3=torch.nn.Dropout2d(p=0.5)

 self.dropout1d=torch.nn.Dropout(p=0.5)

 self.maxpool2d = torch.nn.MaxPool2d(kernel_size = 2, stride = 2, padding = 0)

 self.avgpool2d = torch.nn.AvgPool2d(kernel_size = 2, stride = 2, padding = 0)

 self.fc = torch.nn.Linear(512, 10)

 def forward(self, x):

 ############################# Phase 1
 #print(x.size())
 x = F.relu(self.conv2d_11(x))
 x = self.dropout2d_1(x) #rate =0.3
 x = self.Batchnorm_1(x) #input 64
 #print(x.size())

 x = F.relu(self.conv2d_12(x))
 x = self.dropout2d_1(x) #rate=0.3
 x = self.Batchnorm_1(x) #input 64
 #print(x.size())

 x = self.maxpool2d(x)
 #print(x.size())
 ############################# Phase 2
 x = F.relu(self.conv2d_21(x))
 x = self.dropout2d_1(x) #rate=0.3
 x = self.Batchnorm_2(x) #input 128
 #print(x.size())

 x = F.relu(self.conv2d_22(x))
 x = self.dropout2d_1(x) #rate=0.3
 x = self.Batchnorm_2(x) #input 128
 #print(x.size())

 x = self.maxpool2d(x)
 #print(x.size())
 ############################# Phase 3
 x = F.relu(self.conv2d_31(x))
 x = self.dropout2d_2(x) #rate=0.4
 x = self.Batchnorm_3(x) #input 256
 #print(x.size())

 x = F.relu(self.conv2d_32(x))
 x = self.dropout2d_2(x) #rate=0.4
 x = self.Batchnorm_3(x) #input 256
 #print(x.size())

 x = F.relu(self.conv2d_33(x))
 x = self.dropout2d_2(x) #rate=0.4
 x = self.Batchnorm_3(x) #input 256
 #print(x.size())

 x = self.maxpool2d(x)
 #print(x.size())
 ############################# Phase 4
 x = F.relu(self.conv2d_41(x))
 x = self.dropout2d_2(x)
 x = self.Batchnorm_4(x)
 #print(x.size())

 x = F.relu(self.conv2d_42(x))
 x = self.dropout2d_2(x)
 x = self.Batchnorm_4(x)
 #print(x.size())

 x = self.maxpool2d(x)
 #print(x.size())
 ############################# Phase 5
 x = F.relu(self.conv2d_51(x))
 x = self.dropout2d_3(x)
 x = self.Batchnorm_4(x)
 #print(x.size())

 x = self.avgpool2d(x)
 #print(x.size())
 x = x.view(x.size(0), -1)
 #print(x.size())
 x = self.dropout1d(x)
 x = F.relu(self.fc(x))
 x = self.dropout1d(x)
 #print(x.size())
 x = F.softmax(x)
 ###############################

 return(x)


import time
from torch.optim.lr_scheduler import ReduceLROnPlateau

def trainNet(model, batch_size, n_epochs, learning_rate):

 lr=learning_rate

 #Print all of the hyperparameters of the training iteration:
 print("======= HYPERPARAMETERS =======")
 print("Batch size=", batch_size)
 print("Epochs=", n_epochs)
 print("Base learning_rate=", learning_rate)
 print("=" * 30)

 #Get training data
 n_batches = len(train_loader)

 #Time for printing
 training_start_time = time.time()

 #Loss function"
 loss = torch.nn.CrossEntropyLoss()
 optimizer = createOptimizer(model, lr) 

 scheduler = ReduceLROnPlateau(optimizer, 'min'
 ,patience=3,factor=0.9817
 ,verbose=True,)

 #Loop for n_epochs
 for epoch in range(n_epochs):

 #save the weightsevery 10 epochs
 if epoch % 10 == 0 :
 torch.save(model.state_dict(), 'model.ckpt')


 #print('learning rate : :.3f '.format(lr)) 
 #Create our loss and optimizer functions

 running_loss = 0.0
 print_every = n_batches // 10
 start_time = time.time()
 total_train_loss = 0
 total_train_acc = 0
 epoch_time = 0

 for i, data in enumerate(train_loader, 0):

 #free up the cuda memory
 inputs=None
 labels=None

 inputs, labels = data

 inputs, labels = Variable(inputs.to(device)), Variable(labels.to(device))

 optimizer.zero_grad()

 outputs = model(inputs)

 score, predictions = torch.max(outputs.data, 1)
 acc = (labels==predictions).sum()
 total_train_acc += acc

 loss_size = loss(outputs, labels)
 loss_size.backward()
 optimizer.step()

 running_loss += loss_size.item()
 total_train_loss += loss_size.item()

 #Print every 10th batch of an epoch
 if (i + 1) % (print_every + 1) == 0:
 print("Epoch , :d % t | train_loss: :.3f | train_acc:% | took: :.2fs".format(
 epoch+1, int(100 * (i+1) / n_batches), running_loss / print_every
 ,int(acc), time.time() - start_time))

 epoch_time += (time.time() - start_time)

 #Reset running loss and time
 running_loss = 0.0
 start_time = time.time()

 scheduler.step(total_train_loss)
 torch.cuda.empty_cache() 
 #At the end of the epoch, do a pass on the validation set
 total_val_loss = 0

 for inputs, labels in val_loader:

 #Wrap tensors in Variables
 inputs, labels = Variable(inputs.to(device)), Variable(labels.to(device))

 #Forward pass
 val_outputs = model(inputs)
 val_loss_size = loss(val_outputs, labels)
 total_val_loss += val_loss_size.item()

 print("-"*30)
 print("Train loss = :.2f | Train acc = :.1f% | Val loss=:.2f | took: :.2fs".format(
 total_train_loss / len(train_loader),total_train_acc/ len(train_loader)
 ,total_val_loss/len(val_loader),epoch_time))
 print("="*60)


 print("Training finished, took :.2fs".format(time.time() - training_start_time))
CNN = SimpleCNN().to(device)
CNN.eval()

trainNet(CNN, batch_size=64, n_epochs=250, learning_rate=0.1)

Keras:

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Flatten,Activation
from tensorflow.keras.layers import Conv2D, MaxPool2D,BatchNormalization,GlobalAveragePooling2D

model = Sequential()
#####################################################
# Phase 1
model.add(Conv2D(64,(3,3),input_shape=(32,32,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization())

#(32,32,3)

model.add(Conv2D(64,(3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization())
#(32,32,3)


model.add(MaxPool2D((2,2)))
#(16,16,3)

#####################################################
#Phase 2
model.add(Conv2D(128, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization())
#(16,16,3)

model.add(Conv2D(128, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(BatchNormalization())
#(16,16,3)

model.add(MaxPool2D((2,2),padding='same'))
#(8,8,3)

#####################################################
#Phase 3
model.add(Conv2D(256, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.4))
model.add(BatchNormalization())
#(8,8,3)


model.add(Conv2D(256, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.4))
model.add(BatchNormalization())
#(8,8,3)

model.add(Conv2D(256, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.4))
model.add(BatchNormalization())
#(8,8,3)

model.add(MaxPool2D((2,2)))
#(4,4,3)

#####################################################
#Phase 4
model.add(Conv2D(512, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.4))
model.add(BatchNormalization())
#(4,4,3)

model.add(Conv2D(512, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.4))
model.add(BatchNormalization())
#(4,4,3)

model.add(MaxPool2D((2,2)))
#(2,2,3)

#####################################################
#Phase 5
model.add(Conv2D(512, (3,3),padding='same'))
model.add(Activation('relu'))
model.add(Dropout(rate=0.5))
model.add(BatchNormalization())
#(2,2,3)

model.add(GlobalAveragePooling2D(data_format='channels_last'))
model.add(Flatten())
model.add(Dropout(rate=0.5))

model.add(Dense(10,activation='relu'))
model.add(Dropout(rate=0.5))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer=sgd_optimizer,loss='categorical_crossentropy',metrics=['accuracy'])
history=model.fit(x=x_train,y=y_train,batch_size=64,
 epochs=250,verbose=1,callbacks=[checkpoint],validation_data=(x_test,y_test))

Edit: on a closer look, acc doesn't seem to require gradient, so this paragraph probably doesn't apply.
It looks like the most significant issue is that total_train_acc accumulates history across the training loop (see https://pytorch.org/docs/stable/notes/faq.html for details).
Changing total_train_acc += acc to total_train_acc += acc.item() should fix this.

Another thing you should use with torch.no_grad() for the validation loop.

Not really about speed, but model.train() and model.eval() should be used for training/evaluation to make batchnorm and dropout layers work in correct mode.