AI Blitz XIII
Age prediction - classic classification baseline solution
Data preprocessing with augmentations and fine-tuning DenseNet for classification problems.
trancongthinh
Data preprocessing with augmentations and fine-tuning DenseNet for classification problems.
Download AIcrowd CLI¶
In [2]:
!pip install aicrowd-cli
%load_ext aicrowd.magic
Login to AIcrowd¶
In [3]:
%aicrowd login
Download dataset¶
In [4]:
import os
os.getcwd()
Out[4]:
In [5]:
!mkdir data
%aicrowd ds dl -c age-prediction -o data
In [6]:
!unzip data/train.zip -d data/train > /dev/null
!unzip data/val.zip -d data/val > /dev/null
!unzip data/test.zip -d data/test > /dev/null
Import the libraries¶
In [7]:
from sklearn.model_selection import train_test_split
import pandas as pd
import cv2
from matplotlib import pyplot as plt
import numpy as np
import PIL
from PIL import ImageFont, ImageDraw, Image
import time
import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
Explore the data¶
Read the data¶
In [8]:
train_csv = pd.read_csv(r'/content/data/train.csv')
print("Numer of training input data", len(train_csv))
val_csv = pd.read_csv(r'/content/data/val.csv')
print("Numer of validating input data", len(val_csv))
test_csv = pd.read_csv(r'/content/data/test.csv')
print("Numer of testing input data", len(test_csv))
In [9]:
# A very basic method to create complete image paths :)
for i in range(len(train_csv)):
train_csv.ImageID[i] = train_csv.ImageID[i] + '.jpg'
for i in range(len(val_csv)):
val_csv.ImageID[i] = val_csv.ImageID[i] + '.jpg'
for i in range(len(test_csv)):
test_csv.ImageID[i] = test_csv.ImageID[i] + '.jpg'
print(train_csv.head())
In [10]:
train_csv.age.unique() # number of classifying classes
Out[10]:
In [11]:
def display(im_path):
dpi = 80
im_data = plt.imread(im_path)
height, width = im_data.shape[:2]
# What size does the figure need to be in inches to fit the image?
figsize = width / float(dpi), height / float(dpi)
# Create a figure of the right size with one axes that takes up the full figure
fig = plt.figure(figsize=figsize)
ax = fig.add_axes([0, 0, 1, 1])
# Hide spines, ticks, etc.
ax.axis('off')
# Display the image.
ax.imshow(im_data, cmap='gray')
plt.show()
In [12]:
img_path = '/content/data/train/' + train_csv.ImageID[0]
print(img_path)
display(img_path)
In [13]:
img_path = '/content/data/train/' + train_csv.ImageID[1]
img = cv2.imread(img_path)
print(img.shape) # is it too large for read all image at once?
Data preparation for training and validating purposes¶
In [14]:
class_name = ['30-40', '80-90', '90-100', '40-50', '0-10', '60-70', '70-80', '20-30', '50-60', '10-20']
In [15]:
# standardize images
def prep_fn(img):
img = img.astype(np.float32) / 255.0
img = (img - 0.5) * 2
return img
# data generator function for training and validating purposes with simple augmentation
def data_generator(df_train, df_val, train_data_path, val_data_path, img_size, batch_size, seed):
start_time = time.time()
training_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
preprocessing_function = prep_fn,
horizontal_flip = True,
vertical_flip = True,
brightness_range = [0.8,1.2],
fill_mode = 'nearest')
train_generator = training_datagen.flow_from_dataframe(
dataframe = df_train,
directory = train_data_path,
x_col = "ImageID",
y_col = "age",
target_size = (img_size, img_size),
batch_size = batch_size,
class_mode = 'categorical',
shuffle = False,
seed = seed)
validation_datagen = tf.keras.preprocessing.image.ImageDataGenerator(
preprocessing_function = prep_fn)
val_generator = validation_datagen.flow_from_dataframe(
dataframe = df_val,
directory = val_data_path,
x_col = "ImageID",
y_col = "age",
target_size = (img_size, img_size),
batch_size = batch_size,
class_mode = 'categorical',
shuffle = False,
seed = seed)
print(f'----{time.time()-start_time} seconds----')
return train_generator, val_generator
In [16]:
num_class = 10
img_size = 256
batch_size = 8
seed = 42
lr = 0.02
min_lr = 1e-08
nb_of_epoch = 100
model_name = 'densenet'
In [17]:
train_generator, val_generator = data_generator(df_train = train_csv, df_val = val_csv,
train_data_path = '/content/data/train', val_data_path = '/content/data/val',
img_size = img_size, batch_size = batch_size, seed = seed)
step_per_epoch = train_generator.n//train_generator.batch_size
val_step = val_generator.n//val_generator.batch_size
Create basic tf.keras.models and training¶
In [18]:
def scheduler(epoch):
if epoch<= WARMUP_EPOCH:
lr = INITIAL_LEARNINGRATE *epoch/WARMUP_EPOCH
else:
lr = INITIAL_LEARNINGRATE * DECAY_RATE**(epoch-WARMUP_EPOCH)
return lr
def get_model(fine_tunning = False, checkpoint_path = 'imagenet'):
print('--------------Building The Model...--------------')
base_model = tf.keras.applications.DenseNet121(include_top = False,
weights = checkpoint_path,
input_shape = (img_size, img_size, 3))
base_model.trainable = fine_tunning
print("\nNumber of layers in the base model: ", len(base_model.layers))
x = base_model.output
# x = tf.keras.layers.GlobalMaxPooling2D()(x)
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = tf.keras.layers.Dense(1024,activation='relu')(x)
x = tf.keras.layers.Dense(512,activation='relu')(x)
out = tf.keras.layers.Dense(10, activation='softmax')(x)
model = tf.keras.models.Model(inputs=base_model.input, outputs=out)
model.compile(loss = 'categorical_crossentropy',
optimizer = Adam(learning_rate = lr),
metrics='accuracy')
print('\n--------------Done!--------------')
return model
model = get_model(fine_tunning = False, checkpoint_path = 'imagenet')
In [26]:
def train_model(model, lr, model_name, nb_of_epoch, step_per_epoch, val_step, min_lr, train_generator, val_generator):
#for layer in base_model.layers[:fine_tune_at]:
# layer.trainable = False
print('--------------Deploying the Model...--------------')
monitor = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss',
min_delta = 1e-4,
patience = 5,
verbose = 1,
mode = 'min',
restore_best_weights = True)
lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor = "val_loss",
factor = 0.5,
patience = 2,
verbose = 1,
mode = 'min',
min_delta = 1e-4,
cooldown = 0,
min_lr = min_lr)
filepath = model_name + "-{epoch:02d}-{val_loss:.4f}.hdf5"
checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath,
monitor = 'val_loss',
verbose = 1,
save_best_only = False,
save_weights_only = True,
mode = 'min',
save_freq = 'epoch')
print('--------------Deployed Successfully--------------')
print('--------------Training Begins--------------')
history = model.fit(train_generator,
epochs = nb_of_epoch,
steps_per_epoch = step_per_epoch,
validation_data = val_generator,
verbose = 1,
validation_steps = val_step,
callbacks = [monitor, lr_scheduler, checkpoint])
return history
In [20]:
train_model(model, lr, model_name, nb_of_epoch, step_per_epoch, val_step, min_lr, train_generator, val_generator)
Out[20]:
In [27]:
model.save_weights('best-before-fine-tunning.hdf5')
In [28]:
model = get_model(fine_tunning = True)
checkpoint_path = 'best-before-fine-tunning.hdf5'
model.load_weights(checkpoint_path)
lr = 0.0002
In [29]:
train_model(model, lr, model_name, nb_of_epoch, step_per_epoch, val_step, min_lr, train_generator, val_generator)
Out[29]:
In [30]:
model.save_weights('best-after-fine-tunning.hdf5')
In [32]:
model = get_model(fine_tunning = True)
checkpoint_path = 'best-after-fine-tunning.hdf5'
model.load_weights(checkpoint_path)
Predict test set and submission¶
In [33]:
test_datagen = tf.keras.preprocessing.image.ImageDataGenerator(preprocessing_function = prep_fn)
test_generator = test_datagen.flow_from_dataframe(
dataframe = test_csv,
directory = '/content/data/test',
x_col = "ImageID",
y_col = None,
target_size = (img_size, img_size),
batch_size = batch_size,
class_mode = None,
shuffle = False)
STEP_SIZE_TEST = test_generator.n//test_generator.batch_size
pred = model.predict(test_generator,
steps = STEP_SIZE_TEST,
verbose = 1)
predicted_class_indices = np.argmax(pred, axis = 1)
In [34]:
labels = (train_generator.class_indices)
labels = dict((v,k) for k,v in labels.items())
predictions = [labels[k] for k in predicted_class_indices]
filenames = test_generator.filenames
for i in range(len(filenames)):
filenames[i] = filenames[i][:-4]
results = pd.DataFrame({"ImageID": filenames,
"age": predictions})
In [35]:
results.head()
Out[35]:
In [36]:
!rm -rf assets
!mkdir assets
results.to_csv(os.path.join("assets", "submission.csv"))
In [39]:
%aicrowd notebook submit -c age-prediction -a assets --no-verify
Content
Comments
You must login before you can post a comment.