In [1]:
import os
import math
import matplotlib.pyplot as plt
import numpy as np
from itertools import product
from keras.engine import Model
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.callbacks import ModelCheckpoint, Callback
from keras.preprocessing.image import load_img, img_to_array, array_to_img, ImageDataGenerator
from keras_vggface.vggface import VGGFace
from sklearn.metrics import confusion_matrix
Using TensorFlow backend.

Constants

In [2]:
DATA_DIR = '../data/preprocessed/'
BATCH_SIZE = 32
GRAYSCALE = False
AUGMENTATION_FACTOR = 3
EPOCHS = 100
RANDOM_SEED = 123

Load VGGFace finetuned model

In [3]:
INPUT_DIM = (128, 128, 1 if GRAYSCALE else 3)

def get_model():
    vgg_model = VGGFace(include_top=False, input_shape=INPUT_DIM, pooling='max')

    top_model = Sequential(name='top')
    top_model.add(Dense(128, activation='relu', input_shape=vgg_model.output_shape[1:]))
    top_model.add(Dropout(0.5))
    top_model.add(Dense(4, activation='softmax'))
    
    for layer in vgg_model.layers[:-3]:
        layer.trainable = False
    
    model = Sequential()
    model.add(vgg_model)
    model.add(top_model)
    
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
    
    return model

model = get_model()

model.load_weights('final-22-0.546.hdf5')

Load custom model

In [4]:
INPUT_DIM = (64, 64, 1 if GRAYSCALE else 3)

def get_model():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), activation='relu', input_shape=INPUT_DIM))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(32, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    model.add(Conv2D(32, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    
    model.add(Flatten())
    model.add(Dense(64, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(4, activation='softmax'))
    
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
    return model

model = get_model()

model.load_weights('rgb-28-0.53.hdf5')

Data generator

In [5]:
train_datagen = ImageDataGenerator(
        rotation_range=10,
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)

datagen = ImageDataGenerator(rescale=1./255)

generator_base_params = {
    'target_size': INPUT_DIM[:2],
    'class_mode': 'categorical',
    'color_mode': 'grayscale' if GRAYSCALE else 'rgb',
    'batch_size': BATCH_SIZE,
    'seed': RANDOM_SEED
}

train_generator = train_datagen.flow_from_directory(DATA_DIR + 'train', shuffle=True, **generator_base_params)
test_generator = datagen.flow_from_directory(DATA_DIR + 'test', shuffle=True, **generator_base_params)

n_train = train_generator.n
n_test = test_generator.n
Found 990 images belonging to 4 classes.
Found 235 images belonging to 4 classes.

Investigating the model

In [6]:
test_generator.shuffle = False
test_generator.batch_size = 1
test_generator.reset()

tuples = [test_generator.next() for i in range(n_test)]
x_test, y_real = np.array([x[0][0] for x in tuples]), np.array([x[1][0] for x in tuples])

y_pred = model.predict(x_test)

Class distributions

In [7]:
classes = ['cs', 'econ', 'german', 'mechanical']
dist_real = np.bincount(y_real.argmax(axis=1)) / y_real.shape[0]
dist_pred = np.bincount(y_pred.argmax(axis=1)) / y_pred.shape[0]
print(classes)
print(dist_real)
print(dist_pred)
['cs', 'econ', 'german', 'mechanical']
[0.25106383 0.28085106 0.21276596 0.25531915]
[0.25957447 0.29361702 0.13617021 0.3106383 ]

Examples of false classifications

In [8]:
def print_samples(class_index):
    top_n = 5
    c = class_index
    
    fig = plt.figure(figsize=(8, 8))
    
    idx1 = np.where(y_real[:,c] == 1)
    idx2 = np.where(y_real[:,c] == 0)
    top_false_neg = np.abs((y_real[idx1, c] - y_pred[idx1, c])[0]).argsort()[-top_n:][::-1]
    top_false_pos = np.abs((y_real[idx2, c] - y_pred[idx2, c])[0]).argsort()[-top_n:][::-1]
    top_true_pos = np.abs((y_real[idx1, c] - y_pred[idx1, c])[0]).argsort()[:top_n]

    for i in range(top_n):
        img = array_to_img(x_test[idx1[0][top_false_neg[i]]])
        fig.add_subplot(3, top_n, i + 1)
        plt.imshow(img)
        
    for i in range(top_n):
        img = array_to_img(x_test[idx2[0][top_false_pos[i]]])
        fig.add_subplot(3, top_n, i + 1 + top_n)
        plt.imshow(img)
    
    for i in range(top_n):
        img = array_to_img(x_test[idx1[0][top_true_pos[i]]])
        fig.add_subplot(3, top_n, i + 1 + 2 * top_n)
        plt.imshow(img)
        
    plt.show()

Class: cs

1st row: Top n false negatives (What the model mistakenly thinks computer scientists do NOT look like)

2nd row: Top n false positives (What the model mistakenly thinks computer scientists DO look like)

3rd row: Top n true positives

In [16]:
print_samples(0)

Class: econ

1st row: Top n false negatives (What the model mistakenly thinks economists do NOT look like)

2nd row: Top n false positives (What the model mistakenly thinks economists DO look like)

3rd row: Top n true positives

In [10]:
print_samples(1)

Class: german

1st row: Top n false negatives (What the model mistakenly thinks German linguists do NOT look like)

2nd row: Top n false positives (What the model mistakenly thinks German linguists DO look like)

3rd row: Top n true positives

In [11]:
print_samples(2)

Class: mechanical

1st row: Top n false negatives (What the model mistakenly thinks mechanical engineers do NOT look like)

2nd row: Top n false positives (What the model mistakenly thinks mechanical engineers DO look like)

3rd row: Top n true positives

In [12]:
print_samples(3)