Naver Project (fashion MNIST-tf)

import tensorflow as tf
from tensorflow.keras import layers

from IPython.display import clear_output

from sklearn.model_selection import train_test_split

import time
import numpy as np
import matplotlib.pyplot as plt

tf.__version__

#2.3.0

# Load training and eval data from tf.keras
(train_data, train_labels), (test_data, test_labels) = \
    tf.keras.datasets.fashion_mnist.load_data()

train_data, valid_data, train_labels, valid_labels = \
    train_test_split(train_data, train_labels, test_size=0.1, shuffle=True)

train_data = train_data / 255.
train_data = train_data.reshape(-1, 784)
train_data = train_data.astype(np.float32)
train_labels = train_labels.astype(np.int32)

test_data = test_data / 255.
test_data = test_data.reshape(-1, 784)
test_data = test_data.astype(np.float32)
test_labels = test_labels.astype(np.int32)

valid_data = valid_data / 255.
valid_data = valid_data.reshape(-1, 784)
valid_data = valid_data.astype(np.float32)
valid_labels = valid_labels.astype(np.int32)


print(train_data.shape, train_labels.shape)
print(test_data.shape, test_labels.shape)
print(valid_data.shape, valid_labels.shape)

Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz
32768/29515 [=================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz
26427392/26421880 [==============================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz
8192/5148 [===============================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz
4423680/4422102 [==============================] - 0s 0us/step
(54000, 784) (54000,)
(10000, 784) (10000,)
(6000, 784) (6000,)

def one_hot_label(image, label):
  label = tf.one_hot(label, depth=10)
  return image, label

batch_size = 32
max_epochs = 10

# for train
train_dataset = tf.data.Dataset.from_tensor_slices((train_data, train_labels))
train_dataset = train_dataset.map(one_hot_label)
train_dataset = train_dataset.batch(batch_size=batch_size)
print(train_dataset)

# for test
test_dataset = tf.data.Dataset.from_tensor_slices((test_data, test_labels))
test_dataset = test_dataset.map(one_hot_label)
test_dataset = test_dataset.batch(batch_size=batch_size)
print(test_dataset)

# for test
valid_dataset = tf.data.Dataset.from_tensor_slices((valid_data, valid_labels))
valid_dataset = valid_dataset.map(one_hot_label)
valid_dataset = valid_dataset.batch(batch_size=batch_size)
print(valid_dataset)

#<BatchDataset shapes: ((None, 784), (None, 10)), types: (tf.float32, tf.float32)>
#<BatchDataset shapes: ((None, 784), (None, 10)), types: (tf.float32, tf.float32)>
#<BatchDataset shapes: ((None, 784), (None, 10)), types: (tf.float32, tf.float32)>

model = tf.keras.Sequential()
model.add(layers.Dense(units=128, activation='relu'))
model.add(layers.Dense(units=64, activation='relu'))
model.add(layers.Dense(units=10, activation = 'softmax'))

# without training, just inference a model in eager execution:
for images, labels in train_dataset.take(1):
  predictions = model(images[0:1], training=False)
  print("Predictions: ", predictions.numpy())

#Predictions:  [[0.14850332 0.09409445 0.07706542 0.04800471 0.09944043 0.09219353
#  0.13972872 0.08233118 0.15799843 0.06063977]]

model.summary()

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (1, 128)                  100480    
_________________________________________________________________
dense_1 (Dense)              (1, 64)                   8256      
_________________________________________________________________
dense_2 (Dense)              (1, 10)                   650       
=================================================================
Total params: 109,386
Trainable params: 109,386
Non-trainable params: 0
_________________________________________________________________

# use Adam optimizer 
optimizer = tf.keras.optimizers.Adam(1e-4)
loss_object = tf.keras.losses.CategoricalCrossentropy()
acc_object = tf.keras.metrics.CategoricalAccuracy()

# record loss and accuracy for every epoch
mean_loss = tf.keras.metrics.Mean("loss")
mean_accuracy = tf.keras.metrics.Mean("accuracy")

# save loss and accuracy history for plot
loss_history = []
accuracy_history = [(0, 0.0)]

val_loss_history = []
val_accuracy_history = [(0, 0.0)]

def validation(global_step):
  val_acc_object = tf.keras.metrics.CategoricalAccuracy()

  val_mean_loss = tf.keras.metrics.Mean("loss")
  val_mean_accuracy = tf.keras.metrics.Mean("accuracy")

  for images, labels in valid_dataset:
    predictions = model(images, training=False)
    val_loss_value = loss_object(labels, predictions)
    val_acc_value = val_acc_object(labels, predictions)
    
    val_mean_loss(val_loss_value)
    val_mean_accuracy(val_acc_value)
    
  print("valid loss: {:.4g}, valid accuracy: {:.4g}%".format(val_mean_loss.result(),
                                                             val_mean_accuracy.result() * 100))

  val_loss_history.append((global_step.numpy(), val_mean_loss.result().numpy()))
  val_accuracy_history.append((global_step.numpy(), val_mean_accuracy.result().numpy()))

print("start training!")
global_step = tf.Variable(0, trainable=False)
validation(global_step)
num_batches_per_epoch = int(len(train_data) / batch_size)

for epoch in range(max_epochs):
    
  for step, (images, labels) in enumerate(train_dataset):
    start_time = time.time()
    
    with tf.GradientTape() as tape:
      predictions = model(images, training=True)
      loss_value = loss_object(labels, predictions)
      acc_value = acc_object(labels, predictions)

    grads = tape.gradient(loss_value, model.trainable_variables)
    optimizer.apply_gradients(zip(grads, model.trainable_variables))
    global_step.assign_add(1)
    
    mean_loss(loss_value)
    mean_accuracy(acc_value)
    loss_history.append((global_step.numpy(), mean_loss.result().numpy()))
    
    if global_step.numpy() % 10 == 0:
      clear_output(wait=True)
      epochs = epoch + step / float(num_batches_per_epoch)
      duration = time.time() - start_time
      examples_per_sec = batch_size / float(duration) 
      print("epochs: {:.2f}, step: {}, loss: {:.3g}, accuracy: {:.4g}% ({:.2f} examples/sec; {:.4f} sec/batch)".format(
          epochs, global_step.numpy(), loss_value.numpy(), acc_value.numpy()*100, examples_per_sec, duration))
      
  # save mean accuracy for plot
  accuracy_history.append((global_step.numpy(), mean_accuracy.result().numpy()))
  validation(global_step)
  # clear the history
  mean_accuracy.reset_states()

print("training done!")

#epochs: 10.00, step: 16880, loss: 0.167, accuracy: 85.94% (3103.37 examples/sec; 0.0103 sec/batch)
#valid loss: 0.341, valid accuracy: 87.78%
#training done!

plt.plot(*zip(*loss_history), label='loss')
plt.plot(*zip(*val_loss_history), label='val_loss')
plt.xlabel('Number of steps')
plt.ylabel('Loss value [cross entropy]')
plt.legend()
plt.show()

plt.plot(*zip(*accuracy_history), label='accuracy')
plt.plot(*zip(*val_accuracy_history), label='val_accuracy')
plt.xlabel('Number of steps')
plt.ylabel('Accuracy value')
plt.legend()
plt.show()