# 1. Download Python and the corresponding Anaconda (https://www.anaconda.com/distribution/)
# - Python version could be 2.7 or other version that has available Anaconda in https://www.anaconda.com/distribution/
# - Make sure to select "add python to PATH during the installation"

# 2. Install Tensorflow and Keras in Anaconda
# Not neccessary if you've already installed Keras through R
# Open the Anaconda prompt 
# conda create --name deeplearning python 
# activate deeplearning
# pip install --upgrade tensorflow
# pip install --upgrade keras


# 3. Import libraries and modules
import numpy as np
np.random.seed(123)  # for reproducibility
 
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution2D, MaxPooling2D
from keras.utils import np_utils
from keras.datasets import mnist
from keras import backend as K
K.set_image_dim_ordering('tf')

# 4. Load pre-shuffled MNIST data into train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
 
# 5. Preprocess input data
# For tensorflow, the data format order is width,height,depth (e.g., 28, 28, 1)
#
X_train = X_train.reshape(X_train.shape[0], 28, 28, 1)
X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255 #normalize the data to the range[0,1]
X_test /= 255
 
# 6. Preprocess class labels
# Since the data y_train solely indicates which number the image corresponds to, 
# we need create dummy indicator for the label class. In this example, we have 10 digits (10 label classes).
Y_train = np_utils.to_categorical(y_train, 10) 
Y_test = np_utils.to_categorical(y_test, 10)
 
# 7. Define model architecture
model = Sequential()

model.add(Convolution2D(32, (3, 3), activation='relu', input_shape=(28,28,1), dim_ordering='tf'))
model.add(Convolution2D(32, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
 
model.add(Flatten())
model.add(Dense(128, activation='relu')) # fully-conneced 
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
 
# 8. Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 
# 9. Fit model on training data
#In this stage, the choice of the proportion of validation set is arbitrary
# The nb_epoch is the number of epochs or iterations. In each iteration, we consider 32 training samples at once. 
#Verbose indicates how the progress bar is presented to user during training

history = model.fit(X_train, Y_train, validation_split=0.33, batch_size=32, nb_epoch=10, verbose=1) 
 
# 10. Evaluate model on test data
score = model.evaluate(X_test,Y_test,verbose=0)
print('The test accuracy is {:.4f} and the test loss is {:.4f}'.format(score[0],score[1]))

# Plot the training and validation accuracy and loss among epochs 
# import matplotlib
# matplotlib.use('agg')
# from matplotlib import pyplot as plt 
# plt.plot(history.history['acc'])
# plt.plot(history.history['val_acc'])
# plt.title('model accuracy')
# plt.ylabel('accuracy')
# plt.xlabel('epoch')
# plt.legend(['train', 'validation'], loc='upper left')
# plt.savefig('YOURDIRECTORY/modelaccuracy.pdf')
# plt.plot(history.history['loss'])
# plt.plot(history.history['val_loss'])
# plt.title('model loss')
# plt.ylabel('loss')
# plt.xlabel('epoch')
# plt.legend(['train', 'validation'], loc='upper left')
# plt.savefig('YOURDIRECTORY/modelloss.pdf')