Styjun

I am using Keras embedding layers to create entity embeddings made popular on the Kaggle Rossmann Store Sales 3rd place entry. However, I am unsure about how to map back the embeddings back to the actual categorical values. Let's take a look at a very basic example:

In the code below, I create a dataset with two numeric and one categorical feature.

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from keras.models import Model
from keras.layers import Input, Dense, Concatenate, Reshape, Dropout
from keras.layers.embeddings import Embedding

# create some fake data
data, labels = make_classification(n_classes=2, class_sep=2, n_informative=2,
 n_redundant=0, flip_y=0, n_features=2,
 n_clusters_per_class=1, n_samples=100,
 random_state=10)

cat_col = np.random.choice(a=[0,1,2,3,4], size=100)

data = pd.DataFrame(data)
data[2] = cat_col
embed_cols = [2]

# converting data to list of lists, as the network expects to
# see the data in this format
def preproc(df):
 data_list = []

 # convert cols to list of lists
 for c in embed_cols:
 vals = np.unique(df[c])
 val_map = 
 for i in range(len(vals)):
 val_map[vals[i]] = vals[i]
 data_list.append(df[c].map(val_map).values)

 # the rest of the columns
 other_cols = [c for c in df.columns if (not c in embed_cols)]
 data_list.append(df[other_cols].values)
 return data_list

data = preproc(data)

There are 5 unique values for the categorical column:

print("Unique Values: ", np.unique(data[0]))
Out[01]: array([0, 1, 2, 3, 4])

which then get fed into a Keras model with an embedding layer:

inputs = []
embeddings = []

input_cat_col = Input(shape=(1,))
embedding = Embedding(5, 3, input_length=1, name='cat_col')(input_cat_col)
embedding = Reshape(target_shape=(3,))(embedding)
inputs.append(input_cat_col)
embeddings.append(embedding)


# add the remaining two numeric columns from the 'data array' to the network
input_numeric = Input(shape=(2,))
embedding_numeric = Dense(8)(input_numeric)
inputs.append(input_numeric)
embeddings.append(embedding_numeric)

x = Concatenate()(embeddings)
output = Dense(1, activation='sigmoid')(x)

model = Model(inputs, output)
model.compile(loss='binary_crossentropy', optimizer='adam')

history = model.fit(data, labels,
 epochs=10,
 batch_size=32,
 verbose=1,
 validation_split=0.2)

I can get the actual embeddings by getting the weight for the embedding layer:

embeddings = model.get_layer('cat_col').get_weights()[0]
print("Unique Values: ", np.unique(data[0]))
print("3 Dimensional Embedding: n", embeddings)

Unique Values: [0 1 2 3 4]
3 Dimensional Embedding: 
 [[ 0.02749949 0.04238378 0.0080842 ]
 [-0.00083209 0.01848664 0.0130044 ]
 [-0.02784528 -0.00713446 -0.01167112]
 [ 0.00265562 0.03886909 0.0138318 ]
 [-0.01526615 0.01284053 -0.0403452 ]]

However, I am unsure how to map these back. Is it safe to assume that the weights are ordered? For example, 0=[ 0.02749949 0.04238378 0.0080842 ]?

Yes, the weights of the embedding layer correspond to the word indexed by an integer in the order i.e weight array 0 in the embedding layer correspond to the word with index 0 and so on. You can think of embedding layer as a lookup table where nth row of the table correspond to word vector of the nth word (but embedding layers is trainable layer not just a static lookup table)

inputs = Input(shape=(1,))
embedding = Embedding(5, 3, input_length=1, name='cat_col')(inputs)
model = Model(inputs, embedding)

x = np.array([0,1,2,3,4]).reshape(5,1)
labels = np.zeros((5,1,3))

print (model.predict(x))
print (model.get_layer('cat_col').get_weights()[0])

assert np.array_equal(model.predict(x).reshape(-1), model.get_layer('cat_col').get_weights()[0].reshape(-1))

model.predict(x):

[[[-0.01862894, 0.0021644 , 0.04706952]],
 [[-0.03891206, 0.01743075, -0.03666048]],
 [[-0.01799501, 0.01427511, -0.00056203]],
 [[ 0.03703432, -0.01952349, 0.04562894]],
 [[-0.02806044, -0.04623617, -0.01702447]]]

model.get_layer('cat_col').get_weights()[0]

[[-0.01862894, 0.0021644 , 0.04706952],
 [-0.03891206, 0.01743075, -0.03666048],
 [-0.01799501, 0.01427511, -0.00056203],
 [ 0.03703432, -0.01952349, 0.04562894],
 [-0.02806044, -0.04623617, -0.01702447]]