Loading [MathJax]/jax/output/CommonHTML/fonts/TeX/fontdata.js

Project IA Deep Learning - Supervised

DataSet MNIST Fashion.

Philippe Jadoul - february 2020

Fashion MNIST - Network CNN - Convolution Neural Network

Reseau neuronal convoluty - Wikipedia

https://fr.wikipedia.org/wiki/Réseau_neuronal_convolutif

Imports

import sys
import os
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

Import my_fonctions

sys.path.append("/home/ia-serveur/ia/my_lib_functions")
#sys.path.append("/home/ia/ia/my_lib_functions")
from my_functions import *

sys.path.append('./my_plot_results')
from my_functions_plot import *

Tests All

assert hasattr(tf, "function") # Be sure to use tensorflow 2.X
print('tensorflow :',tf.__version__)
first_function()

tensorflow : 2.2.0
This is the first function

Load the dataset: Fashion MNIST

fashion_mnist = keras.datasets.fashion_mnist
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

print(train_images.shape, train_labels.shape )
print(test_images.shape, test_labels.shape )

(60000, 28, 28) (60000,)
(10000, 28, 28) (10000,)

max = 0
if max > 0 :
    train_images = train_images[:max]
    train_labels = train_labels[:max]
else:
    train_images = train_images
    train_labels = train_labels

Convert dataset Train - Test to float

train_images = train_images.astype(np.float32) 
test_images = test_images.astype(np.float32) 

Shuffle dataset Train

indexes = np.arange(train_images.shape[0])
np.random.shuffle(indexes)
train_images = train_images[indexes]
train_labels = train_labels[indexes]

Create Validation Dataset

train_images, train_images_validate, train_labels, train_labels_validate = train_test_split(
                                                    train_images, train_labels, test_size=0.2, random_state=None)

print("train_images.shape", train_images.shape)
print("train_labels.shape", train_labels.shape)
print("train_images_validate.shape", train_images_validate.shape)
print("train_labels_validate.shape", train_labels_validate.shape)

train_images.shape (48000, 28, 28)
train_labels.shape (48000,)
train_images_validate.shape (12000, 28, 28)
train_labels_validate.shape (12000,)

List targets_names - Labels og Classes

targets_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

To verify that the data is in the correct format and that you're ready to build and train the network, let's display the first 25 images from the training set and display the class name below each image.

plt.figure(figsize=(10,10))
for i in range(25):
    plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(targets_names[train_labels[i]])
plt.show()

plot images

plt.figure(figsize=(12, 4))

plt.subplot(1, 3, 1)
plt.title('train_images[0]')
plt.imshow(np.reshape(train_images[0], (28, 28)), cmap=plt.cm.binary)
plt.colorbar()
plt.grid(False)
plt.xlabel(targets_names[train_labels[0]])

plt.subplot(1, 3, 2)
plt.title('train_images_validate[0]')
plt.imshow(np.reshape(train_images_validate[0], (28, 28)), cmap=plt.cm.binary)
plt.colorbar()
plt.grid(False)
plt.xlabel(targets_names[train_labels_validate[0]])

plt.subplot(1, 3, 3)
plt.title('test_images[0]')
plt.imshow(np.reshape(test_images[0], (28, 28)), cmap=plt.cm.binary)
plt.colorbar()
plt.grid(False)
plt.xlabel(targets_names[test_labels[0]])

plt.show()

Normalization - StandardScater - $z = \frac{(X - \overline{U})}{\sigma}$

• Le score standard d'un echantillon X est calcule comme suit :
  • U est la moyenne des echantillons d'apprentisage.
  • $\sigma$ est l'ecart type des echantillons d'apprentisage
    
    
• fit_transform computes the mean and std to be used for later scaling.
• transform usese a previously computed mean an std.
• but first fit_transform does both at the same time

print("Before normalization (train_images)- Mean of data =", train_images.mean(),"- std of date =",
      train_images.std())
print("Before normalization (train_images_validate)- Mean of data =", train_images_validate.mean(),
      "- std of date =",train_images_validate.std())
print("Before normalization (train_test)- Mean of data =", test_images.mean(),
      "- std of date =",test_images.std())

scaler = StandardScaler()
scaler_train_images = scaler.fit_transform(train_images.reshape(-1, 28*28))
scaler_train_images_validate = scaler.transform(train_images_validate.reshape(-1, 28*28))
scaler_test_images = scaler.transform(test_images.reshape(-1, 28*28))

print("After normalization (scater_train_images)- Mean of data =", scaler_train_images.mean(),"- std of date =", 
      scaler_train_images.std())
print("After normalization (scater_train_images_validate)- Mean of data =", scaler_train_images_validate.mean(),
      "- std of date =", scaler_train_images_validate.std())
print("After normalization (scater_test_images)- Mean of data =", scaler_test_images.mean(),
      "- std of date =", scaler_test_images.std())

Before normalization (train_images)- Mean of data = 72.99627 - std of date = 90.058426
Before normalization (train_images_validate)- Mean of data = 72.71696 - std of date = 89.87177
Before normalization (train_test)- Mean of data = 73.146576 - std of date = 89.873276
After normalization (scater_train_images)- Mean of data = -4.0547378e-11 - std of date = 0.99999964
After normalization (scater_train_images_validate)- Mean of data = -0.0039417264 - std of date = 0.9931064
After normalization (scater_test_images)- Mean of data = 0.00171644 - std of date = 1.00799

Distribution Data

plt.figure(figsize=(16, 4))

plt.subplot(1, 3, 1)
plt.hist(scaler_train_images[0].ravel(), color='gray', bins=100)
plt.title('Histogram Distribution Data')
plt.xlabel('Valeur Data - scaler_train_images[0]')
plt.ylabel('Frequency')
plt.grid(True)

plt.subplot(1, 3, 2)
plt.hist(scaler_train_images_validate[0].ravel(), color='gray', bins=100)
plt.title('Histogram Distribution Data')
plt.xlabel('Valeur Data - scaler_train_images_validate[0]')
plt.ylabel('Frequency')
plt.grid(True)

plt.subplot(1, 3, 3)
plt.hist(scaler_test_images[0].ravel(), color='gray', bins=100)
plt.title('Histogram Distribution Data')
plt.xlabel('Valeur Data - scaler_test_images[0]')
plt.ylabel('Frequency')
plt.grid(True)

plt.show()

in a convolution model the datas must be tensors - Data img (x,y,z) - z=1 for image grey - z=3 for image RGB

scaled_train_images = scaler_train_images.reshape(-1, 28, 28, 1)
scaled_train_images_validate = scaler_train_images_validate.reshape(-1, 28, 28, 1)
scaled_test_images = scaler_test_images.reshape(-1, 28, 28, 1)
print('scaled_train_images.shape :', scaled_train_images.shape)
print('scaled_train_images_validate.shape :', scaled_train_images_validate.shape)
print('scaled_test_images.shape :', scaled_test_images.shape)

scaled_train_images.shape : (48000, 28, 28, 1)
scaled_train_images_validate.shape : (12000, 28, 28, 1)
scaled_test_images.shape : (10000, 28, 28, 1)

one-hot encoding

unique, counts = np.unique(train_labels, return_counts=True)
n_classes = len(unique)

Y_train_labels = tf.keras.utils.to_categorical(train_labels, n_classes )
Y_train_labels_validate = tf.keras.utils.to_categorical(train_labels_validate, n_classes )
Y_test_labels = tf.keras.utils.to_categorical(test_labels, n_classes )

print('Diff Classes :',unique)
print('nb Classes :',n_classes)
print('Shape after one-hot encoding :', Y_train_labels.shape)
print('Labels [',train_labels[0],']Y_Train_labels :',Y_train_labels[0])
print('Labels [',train_labels_validate[0],']Y_Train_labels_validate :',Y_train_labels_validate[0])
print('Labels [',test_labels[0],']Y_Test_labels  :',Y_test_labels[0])

Diff Classes : [0 1 2 3 4 5 6 7 8 9]
nb Classes : 10
Shape after one-hot encoding : (48000, 10)
Labels [ 7 ]Y_Train_labels : [0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]
Labels [ 4 ]Y_Train_labels_validate : [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]
Labels [ 9 ]Y_Test_labels  : [0. 0. 0. 0. 0. 0. 0. 0. 0. 1.]

Create the training dataset - Batch

X_train_images = tf.data.Dataset.from_tensor_slices((scaled_train_images, Y_train_labels))
X_train_images_validate = tf.data.Dataset.from_tensor_slices((scaled_train_images_validate, 
                                                              Y_train_labels_validate))
X_test_images = tf.data.Dataset.from_tensor_slices((scaled_test_images, Y_test_labels))
X1_test_images = scaler_test_images

Test Iter in dataset with epoch and batch size

# Iter in the dataset with a number of epoch and batch size
epoch = 1
batch_size = 32
for images_batch, targets_batch in X_train_images.repeat(epoch).batch(batch_size):
    print(images_batch.shape, targets_batch.shape)
    break

(32, 28, 28, 1) (32, 10)

Model Draph

Create the CNN model

• layer Convolution.
  • filters : number of output filters in the convolution.
  • kernel_size : convolution window.
  • strides : steps of the convolution.
    
• layer Maxpool2D.
  • sample-based discretization process.
    
• layer Dropout.
  • regularization technique for reducing overfitting.
• layer Flatten.
  • convert outout convokution to single vector.
• layer Dense1.
  • connect ouput Flatten to conventional network.
• layer Dense2.
  • output layer for num of class

# define model cnnn
model = tf.keras.models.Sequential()
# Conv1
model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), strides=1,input_shape=(28,28,1,),
                                 padding='Same', activation="relu"))
model.add(tf.keras.layers.MaxPool2D((2,2)))
model.add(tf.keras.layers.Dropout(0.25))
# Conv2
model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=(5,5), strides=2, padding='Same', activation="relu"))
model.add(tf.keras.layers.MaxPool2D((2,2)))
model.add(tf.keras.layers.Dropout(0.25))
# Conv3
model.add(tf.keras.layers.Conv2D(filters=128, kernel_size=(5,5), strides=2, padding='Same', activation="relu"))
model.add(tf.keras.layers.MaxPool2D((2,2)))
model.add(tf.keras.layers.Dropout(0.25))
# flatten
model.add(tf.keras.layers.Flatten())
# dense 1
model.add(tf.keras.layers.Dense(128, activation="relu"))
model.add(tf.keras.layers.Dropout(0.2))
# dense 2
model.add(tf.keras.layers.Dense(64, activation="relu"))
model.add(tf.keras.layers.Dropout(0.4))
# output
model.add(tf.keras.layers.Dense(10, activation="softmax"))

Model Summary

model.summary()

Model: "sequential_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_9 (Conv2D)            (None, 28, 28, 32)        832       
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 14, 14, 32)        0         
_________________________________________________________________
dropout_15 (Dropout)         (None, 14, 14, 32)        0         
_________________________________________________________________
conv2d_10 (Conv2D)           (None, 7, 7, 64)          51264     
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 3, 3, 64)          0         
_________________________________________________________________
dropout_16 (Dropout)         (None, 3, 3, 64)          0         
_________________________________________________________________
conv2d_11 (Conv2D)           (None, 2, 2, 128)         204928    
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 1, 1, 128)         0         
_________________________________________________________________
dropout_17 (Dropout)         (None, 1, 1, 128)         0         
_________________________________________________________________
flatten_3 (Flatten)          (None, 128)               0         
_________________________________________________________________
dense_9 (Dense)              (None, 128)               16512     
_________________________________________________________________
dropout_18 (Dropout)         (None, 128)               0         
_________________________________________________________________
dense_10 (Dense)             (None, 64)                8256      
_________________________________________________________________
dropout_19 (Dropout)         (None, 64)                0         
_________________________________________________________________
dense_11 (Dense)             (None, 10)                650       
=================================================================
Total params: 282,442
Trainable params: 282,442
Non-trainable params: 0
_________________________________________________________________

Compiling the sequentail model - CNN

• Loss function —This measures how accurate the model is during training with (spare_categorical_crossentropy)
  • (categorical) - possibilité d'avoir plus de deux classe.
  • (spare) - un seul entier par classe.
• opimizer function —This is how the model is updated based on the data it sees and its loss function.
  • (Adam) - Adaptive Moment Estimation - minimize the function with method Stochastic Gradien Descent.

• Metrics — Used to monitor the training and testing steps.

  • (accuracy) - Precision (also called predictve value)

• Si les classes ou Labels sont code sur un digit (ex: 9 -> [9] alors

  • loss='sparse_categorical_crossentropy'
• Si les classes ou Labels sont code en One-hot encoding (ex: 9 -> [0,0,0,0,0,0,0,0,1] alors
  • loss='"categorical_crossentropy'

Function Compile Model

# Compile the model
def compile_model():
    model.compile(
        loss="categorical_crossentropy",
        optimizer="adam",
        metrics=["accuracy"]
    )

Compile Model

compile_model()

Function Save Model

def save_model(model_name,model_format):
    path = './results'
    model_path = os.path.join(path, model_name)
    model.save(model_path, save_format=model_format)

Save the Compiled Model

model_name = 'mnist_cnn2_base.h5'
model_format ='h5'
save_model(model_name, model_format)

model_name = 'mnist_cnn2_acc.h5'
model_format ='h5'
save_model(model_name, model_format)

model_name = 'mnist_cnn2_loss.h5'
model_format ='h5'
save_model(model_name, model_format)

train set with batch dataset

History model fit

history = model.fit(X_train_images.batch(100), epochs=50, verbose=False)

history = model.fit(X_train_images.batch(100), epochs=50, verbose=False)

Graph - fig-1

plotting_train_metrics(history, save=False, name='fig-1.png')

Train max accuracy = 93.70 %
Train max loss = 18.12 %

Reload the Compiled Model Base - mnist_cnn2_base.h5

del model
name = './results/mnist_cnn2_base.h5'
#model = tf.keras.models.load_model(name)
model = load_model(name)

validation_data with fit function

• dataset : from tf.data structure - with batch.
• epochs : value of repeat in dataset.
• validation_data : from tf.data structue - with batch.
• validation_steps : validation only on a specific number of batches from validation dataset.
• callbacks : ModelCheckpoint allow us to extract the best end-of-epoch model.

BT    = 32
EP    = 50
SP    = None
Callback_list   = None

history model fit - (BT=32, EP=50, SP=None, callback_list=None)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT), validation_steps=SP, callbacks=Callback_list , verbose=True)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT),
                   validation_steps=SP, callbacks=Callback_list , verbose=False)

Graph - fig-2

plotting_train_metrics(history, save=False, name='fig-2.png')

Train max accuracy = 92.45 %  <--> Train max val_accuracy = 91.77 %
Train max loss     = 21.69 %  <--> Train max val_loss     = 23.54 %

BT    = 128
EP    = 50
SP    = None
Callback_list   = None

Reload the Compiled Model Base - mnist_cnn2_base.h5

del model
name = './results/mnist_cnn2_base.h5'
#model = tf.keras.models.load_model(name)
model = load_model(name)

history model fit - (BT=128, EP=50, SP=None, callback_list=None)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT), validation_steps=SP, callbacks=Callback_list , verbose=True)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT),
                   validation_steps=SP, callbacks=Callback_list , verbose=False)

Graph - fig-3

plotting_train_metrics(history, save=False, name='fig-3.png')

Train max accuracy = 93.72 %  <--> Train max val_accuracy = 92.18 %
Train max loss     = 17.48 %  <--> Train max val_loss     = 22.95 %

Re-Load the Compiled Model Base - mnist_cnn2_base.h5

del model
name = './results/mnist_cnn2_base.h5'
#model = tf.keras.models.load_model(name)
model = load_model(name)

Optimization With Max Accuracy & Save Model - mnist_cnn2_acc.h5

Callback_list - val_accuracy , max

ModelCheckpoint allow us to extract the best end-of-epoch model. Under different circumstance, we might monitor val_loss or val_acc

• Callback_list = [ModelCheckpoint(filepath='./results/mnist_cnn2_acc.h5', monitor='val_accuracy',save_best_only=True,mode='max')]

BT    = 128
EP    = 50
SP    = None

Callback_list  = [ModelCheckpoint(filepath='./results/mnist_cnn2_acc.h5', monitor='val_accuracy', 
                                                                save_best_only=True,mode='max')]

history model fit with callbacks (BT=128, EP=50, SP=None, callback_list)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT), validation_steps=SP, callbacks=Callback_list , verbose=True)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT),
                   validation_steps=SP, callbacks=Callback_list , verbose=False)  

function plotting the metrics of train dataset accuracy & loss

• function -plotting_train_metrics(history, save=False, name='fig-.png')

Graph - Best Acc

plotting_train_metrics(history, save=False, name='fig-best-acc.png')

Train max accuracy = 93.90 %  <--> Train max val_accuracy = 92.27 %
Train max loss     = 17.10 %  <--> Train max val_loss     = 23.36 %

Re-Load the Compiled Model Base - mnist_cnn2_base.h5

del model
name = './results/mnist_cnn2_base.h5'
#model = tf.keras.models.load_model(name)
model = load_model(name)

Optimization With Min Loss & Save Model - mnist_cnn2_loss.h5

Callback_list - val_loss , min

ModelCheckpoint allow us to extract the best end-of-epoch model. Under different circumstance, we might monitor val_loss or val_acc

• Callback_list = [ModelCheckpoint(filepath='./results/mnist_cnn2_loss.h5', monitor='val_loss',save_best_only=True,mode='min')]

BT    = 128
EP    = 50
SP    = None

Callback_list  = [ModelCheckpoint(filepath='./results/mnist_cnn2_loss.h5', monitor='val_loss', 
                                                                save_best_only=True,mode='min')]

history model fit with callbacks (BT=128, EP=50, SP=None, callback_list)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT), validation_steps=SP, callbacks=Callback_list , verbose=True)

history = model.fit(X_train_images.batch(BT), epochs=EP, validation_data=X_train_images_validate.batch(BT),
                   validation_steps=SP, callbacks=Callback_list , verbose=False) 

function plotting the metrics of train dataset accuracy & loss

• function -plotting_train_metrics(history, save=False, name='fig-.png')

Graph - Best Loss

plotting_train_metrics(history, save=False, name='fig-best-loss.png')

Train max accuracy = 93.61 %  <--> Train max val_accuracy = 92.04 %
Train max loss     = 17.58 %  <--> Train max val_loss     = 24.33 %

Delete the model

del model

Load the Model

#model = tf.keras.models.load_model('./results/mnist_cnn2_acc.h5')
name = './results/mnist_cnn2_acc.h5'
model = load_model(name)

Model Summary

model.summary()

Model: "sequential_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_9 (Conv2D)            (None, 28, 28, 32)        832       
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 14, 14, 32)        0         
_________________________________________________________________
dropout_15 (Dropout)         (None, 14, 14, 32)        0         
_________________________________________________________________
conv2d_10 (Conv2D)           (None, 7, 7, 64)          51264     
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 3, 3, 64)          0         
_________________________________________________________________
dropout_16 (Dropout)         (None, 3, 3, 64)          0         
_________________________________________________________________
conv2d_11 (Conv2D)           (None, 2, 2, 128)         204928    
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 1, 1, 128)         0         
_________________________________________________________________
dropout_17 (Dropout)         (None, 1, 1, 128)         0         
_________________________________________________________________
flatten_3 (Flatten)          (None, 128)               0         
_________________________________________________________________
dense_9 (Dense)              (None, 128)               16512     
_________________________________________________________________
dropout_18 (Dropout)         (None, 128)               0         
_________________________________________________________________
dense_10 (Dense)             (None, 64)                8256      
_________________________________________________________________
dropout_19 (Dropout)         (None, 64)                0         
_________________________________________________________________
dense_11 (Dense)             (None, 10)                650       
=================================================================
Total params: 282,442
Trainable params: 282,442
Non-trainable params: 0
_________________________________________________________________

Evaluate the Models' Performance with the best current metrics - model mnist_cnn2_acc.h5

compare how the model performs on the test dataset:

• rem : Uniquement sur Images test car le label résultat n'est évidement pas connu
• results = model.evaluate(X_test_images.batch(BT), verbose=True)

BT = 128

#test_loss, test_acc = model.evaluate(X_test_images.batch(BT), verbose=False)
results = model.evaluate(X_test_images.batch(BT), verbose=True)
print('test_loss, test_acc', results)
print('test_accuracy = {t1:3.2f} %'.format(t1=results[1]*100.0))
print('test_loss     = {t1:3.2f} %'.format(t1=results[0]*100.0))

79/79 [==============================] - 2s 21ms/step - loss: 0.2508 - accuracy: 0.9188
test_loss, test_acc [0.2508201599121094, 0.9187999963760376]
test_accuracy = 91.88 %
test_loss     = 25.08 %

Make predictions

With the model trained, you can use it to make predictions about some images.

predictions = model.predict(X_test_images.batch(BT), verbose=False)
predictions.shape

(10000, 10)

accuracy of the prediction in % - img 23

print('validated at = {t:3.2f} %'.format(t= np.max(predictions[23]*100)))
print('predict Class = ', np.argmax(predictions[23]),' - true Class is = ', np.argmax(Y_test_labels[23]))

validated at = 61.43 %
predict Class =  9  - true Class is =  9

i = 23
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plt.title('img')
plot_image(i, predictions[i], Y_test_labels, X1_test_images, targets_names)
plt.subplot(1,2,2)
plt.title('Class')
plot_value_array(i, predictions[i], Y_test_labels) #test_labels)
plt.show()

import fonction plotresults - my library

• function - plot_image(i, predictions[i], Y_test, X_test, targets_names)
• function plot_value_array(i, predictions[i], Y_test) #test_labels)
• Correct prediction labels are blue and incorrect prediction labels are red.
• The number gives the percentage for the predicted label.

Let's plot several images with their predictions. Note that the model can be wrong even when very confident.

# Plot the first X test images, their predicted labels, and the true labels.
# Color correct predictions in blue and incorrect predictions in red.
num_rows = 5
num_cols = 3
num_images = num_rows*num_cols
plt.figure(figsize=(2*2*num_cols, 2*num_rows))
for i in range(num_images):
  plt.subplot(num_rows, 2*num_cols, 2*i+1)
  plot_image(i, predictions[i], Y_test_labels, X1_test_images, targets_names)
  plt.subplot(num_rows, 2*num_cols, 2*i+2)
  plot_value_array(i, predictions[i], Y_test_labels)
plt.tight_layout()
plt.show()

Where Does the Model Make Mistakes ? - Statistics -

• function confusion_matrix from sklearn.metrics
• Compute confusion matrix to evaluate the accuracy of classification

mistakes_stat(Y_test_labels, predictions, targets_names, board=True)

	Label Class	Label Name	Max result	max Class in data	valideted %
0	0	T-shirt/top	860	1000	86.0
1	1	Trouser	987	1000	98.7
2	2	Pullover	889	1000	88.9
3	3	Dress	920	1000	92.0
4	4	Coat	890	1000	89.0
5	5	Sandal	971	1000	97.1
6	6	Shirt	734	1000	73.4
7	7	Sneaker	984	1000	98.4
8	8	Bag	986	1000	98.6
9	9	Ankle boot	967	1000	96.7

Delete the model

del model

Load the Model

#model = tf.keras.models.load_model('./results/mnist_cnn2_loss.h5')
name = './results/mnist_cnn2_loss.h5'
model = load_model(name)

Model Summary

model.summary()

Model: "sequential_3"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_9 (Conv2D)            (None, 28, 28, 32)        832       
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 14, 14, 32)        0         
_________________________________________________________________
dropout_15 (Dropout)         (None, 14, 14, 32)        0         
_________________________________________________________________
conv2d_10 (Conv2D)           (None, 7, 7, 64)          51264     
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 3, 3, 64)          0         
_________________________________________________________________
dropout_16 (Dropout)         (None, 3, 3, 64)          0         
_________________________________________________________________
conv2d_11 (Conv2D)           (None, 2, 2, 128)         204928    
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 1, 1, 128)         0         
_________________________________________________________________
dropout_17 (Dropout)         (None, 1, 1, 128)         0         
_________________________________________________________________
flatten_3 (Flatten)          (None, 128)               0         
_________________________________________________________________
dense_9 (Dense)              (None, 128)               16512     
_________________________________________________________________
dropout_18 (Dropout)         (None, 128)               0         
_________________________________________________________________
dense_10 (Dense)             (None, 64)                8256      
_________________________________________________________________
dropout_19 (Dropout)         (None, 64)                0         
_________________________________________________________________
dense_11 (Dense)             (None, 10)                650       
=================================================================
Total params: 282,442
Trainable params: 282,442
Non-trainable params: 0
_________________________________________________________________

Evaluate the Models' Performance with the best current metrics - model mnist_cnn2_loss.h5

compare how the model performs on the test dataset:

• rem : Uniquement sur Images test car le label résultat n'est évidement pas connu
• results = model.evaluate(X_test_images.batch(BT), verbose=True)

BT = 128

#test_loss, test_acc = model.evaluate(X_test_images.batch(BT), verbose=False)
results = model.evaluate(X_test_images.batch(BT), verbose=True)
print('test_loss, test_acc', results)
print('test_accuracy = {t1:3.2f} %'.format(t1=results[1]*100.0))
print('test_loss     = {t1:3.2f} %'.format(t1=results[0]*100.0))

79/79 [==============================] - 2s 20ms/step - loss: 0.2496 - accuracy: 0.9153
test_loss, test_acc [0.24955114722251892, 0.9153000116348267]
test_accuracy = 91.53 %
test_loss     = 24.96 %

Make predictions

With the model trained, you can use it to make predictions about some images.

predictions = model.predict(X_test_images.batch(BT), verbose=False)
predictions.shape

(10000, 10)

accuracy of the prediction in %

print('validated at = {t:3.2f} %'.format(t= np.max(predictions[23]*100)))
print('predict Class = ', np.argmax(predictions[23]),' - true Class is = ', np.argmax(Y_test_labels[23]))

validated at = 99.62 %
predict Class =  5  - true Class is =  9

i = 23
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plt.title('img')
plot_image(i, predictions[i], Y_test_labels, X1_test_images, targets_names)
plt.subplot(1,2,2)
plt.title('Class')
plot_value_array(i, predictions[i], Y_test_labels) #test_labels)
plt.show()

import fonction plotresults - my library

• function - plot_image(i, predictions[i], Y_test, X_test, targets_names)
• function plot_value_array(i, predictions[i], Y_test) #test_labels)
• Correct prediction labels are blue and incorrect prediction labels are red.
• The number gives the percentage for the predicted label.

Let's plot several images with their predictions. Note that the model can be wrong even when very confident.

# Plot the first X test images, their predicted labels, and the true labels.
# Color correct predictions in blue and incorrect predictions in red.
num_rows = 5
num_cols = 3
num_images = num_rows*num_cols
plt.figure(figsize=(2*2*num_cols, 2*num_rows))
for i in range(num_images):
  plt.subplot(num_rows, 2*num_cols, 2*i+1)
  plot_image(i, predictions[i], Y_test_labels, X1_test_images, targets_names)
  plt.subplot(num_rows, 2*num_cols, 2*i+2)
  plot_value_array(i, predictions[i], Y_test_labels)
plt.tight_layout()
plt.show()

Where Does the Model Make Mistakes ? - Statistics -

• function confusion_matrix from sklearn.metrics
• Compute confusion matrix to evaluate the accuracy of classification

mistakes_stat(Y_test_labels, predictions, targets_names, board=True)

	Label Class	Label Name	Max result	max Class in data	valideted %
0	0	T-shirt/top	864	1000	86.4
1	1	Trouser	978	1000	97.8
2	2	Pullover	889	1000	88.9
3	3	Dress	927	1000	92.7
4	4	Coat	890	1000	89.0
5	5	Sandal	969	1000	96.9
6	6	Shirt	713	1000	71.3
7	7	Sneaker	989	1000	98.9
8	8	Bag	984	1000	98.4
9	9	Ankle boot	950	1000	95.0