Step (1) Import Libraries Step (2) Visualize some Images from Dataset Step (3) Detect Face from Images Step (4) Detected Eyes from Images Step (5) Data Pre-Processing Step (6) Split Data into Training Features and Output Labels Step (7) Reshape Training Features Step (8) Label Encoding Step (9) Train Test Split Step (10) Data Augmentation Step (11) Build Model Step (12) Train Model Step (13) Save Model Step (14) Evaluate Performance Step (15) Live Prediction
# Import Libraries
import os
import cv2
import time
import warnings
import numpy as np
import pandas as pd
import seaborn as sns
import tensorflow as tf
import matplotlib.pyplot as plt
plt.style.use('fivethirtyeight')
warnings.filterwarnings("ignore")
from tensorflow.keras.models import Model
from tensorflow.keras.models import Sequential
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.layers import Input, Lambda, Dense, Flatten, Conv2D, MaxPooling2D, Dropout
from sklearn.metrics import accuracy_score, confusion_matrix, balanced_accuracy_score, classification_report
# Visualize Images
fig=plt.figure(figsize=(15, 15))
columns = 4
rows = 1
for i in range(1, columns*rows +1):
img = plt.imread("Dataset/Closed/_"+str(i)+".jpg")
fig.add_subplot(rows, columns, i)
plt.title("Closed Eyes",color="black")
plt.imshow(img)
plt.xticks([]), plt.yticks([])
# Visualize Images
fig=plt.figure(figsize=(15, 15))
columns = 4
rows = 1
for i in range(1, columns*rows +1):
img = plt.imread("Dataset/Open/_"+str(i)+".jpg")
fig.add_subplot(rows, columns, i)
plt.title("Open Eyes",color="black")
plt.imshow(img)
plt.xticks([]), plt.yticks([])
# Visualize Images
fig=plt.figure(figsize=(15, 15))
columns = 4
rows = 1
for i in range(1, columns*rows +1):
img = plt.imread("Dataset/Yawn/"+str((i+1)*20)+".jpg")
fig.add_subplot(rows, columns, i)
plt.title("Yawn",color="black")
plt.imshow(img)
plt.xticks([]), plt.yticks([])
# Visualize Images
fig=plt.figure(figsize=(15, 15))
columns = 4
rows = 1
for i in range(1, columns*rows +1):
img = plt.imread("Dataset/No_Yawn/"+str((i+1))+".jpg")
fig.add_subplot(rows, columns, i)
plt.title("No_Yawn",color="black")
plt.imshow(img)
plt.xticks([]), plt.yticks([])
%%time
# Detect Faces
def face_for_yawn(direc="Dataset", face_cas_path="Dataset/face.xml"):
yaw_no = []
IMG_SIZE = 145
categories = ["Yawn", "No_Yawn"]
for category in categories:
path_link = os.path.join(direc, category)
class_num1 = categories.index(category)
for image in os.listdir(path_link):
image_array = cv2.imread(os.path.join(path_link, image), cv2.IMREAD_COLOR)
face_cascade = cv2.CascadeClassifier(face_cas_path)
faces = face_cascade.detectMultiScale(image_array, 1.3, 5)
for (x, y, w, h) in faces:
img = cv2.rectangle(image_array, (x, y), (x+w, y+h), (0, 255, 0), 2)
roi_color = img[y:y+h, x:x+w]
resized_array = cv2.resize(roi_color, (IMG_SIZE, IMG_SIZE))
yaw_no.append([resized_array, class_num1])
return yaw_no
yawn_no_yawn = face_for_yawn()
Wall time: 1min 50s
%%time
def get_data(dir_path="Dataset/", face_cas="Face.xml", eye_cas="Eye.xml"):
labels = ['Closed', 'Open']
IMG_SIZE = 145
data = []
for label in labels:
path = os.path.join(dir_path, label)
class_num = labels.index(label)
class_num +=2
for img in os.listdir(path):
try:
img_array = cv2.imread(os.path.join(path, img), cv2.IMREAD_COLOR)
resized_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
data.append([resized_array, class_num])
except Exception as e:
print(e)
return data
Wall time: 0 ns
%%time
# Append Data
def append_data():
yaw_no = face_for_yawn()
data = get_data()
yaw_no.extend(data)
return np.array(yaw_no)
processed_data = append_data()
Wall time: 2min 7s
# Split Data
X = []
y = []
for feature, label in processed_data:
X.append(feature)
y.append(label)
# Reshape
X = np.array(X)
X = X.reshape(-1, 145, 145, 3)
# Label Encoding
label_bin = LabelBinarizer()
y = label_bin.fit_transform(y)
y = np.array(y)
# Train Test Split
seed = 42
test_size = 0.20
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=seed, test_size=test_size)
# Data Augmentation
train_generator = ImageDataGenerator(rescale=1/255, zoom_range=0.2, horizontal_flip=True, rotation_range=30)
test_generator = ImageDataGenerator(rescale=1/255)
train_generator = train_generator.flow(np.array(X_train), y_train, shuffle=False)
test_generator = test_generator.flow(np.array(X_test), y_test, shuffle=False)
# Build Model
model = Sequential()
model.add(Conv2D(256, (3, 3), activation="relu", input_shape=X_train.shape[1:]))
model.add(MaxPooling2D(2, 2))
model.add(Conv2D(128, (3, 3), activation="relu"))
model.add(MaxPooling2D(2, 2))
model.add(Conv2D(64, (3, 3), activation="relu"))
model.add(MaxPooling2D(2, 2))
model.add(Conv2D(32, (3, 3), activation="relu"))
model.add(MaxPooling2D(2, 2))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(Dense(64, activation="relu"))
model.add(Dense(4, activation="softmax"))
model.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 143, 143, 256) 7168 _________________________________________________________________ max_pooling2d (MaxPooling2D) (None, 71, 71, 256) 0 _________________________________________________________________ conv2d_1 (Conv2D) (None, 69, 69, 128) 295040 _________________________________________________________________ max_pooling2d_1 (MaxPooling2 (None, 34, 34, 128) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 32, 32, 64) 73792 _________________________________________________________________ max_pooling2d_2 (MaxPooling2 (None, 16, 16, 64) 0 _________________________________________________________________ conv2d_3 (Conv2D) (None, 14, 14, 32) 18464 _________________________________________________________________ max_pooling2d_3 (MaxPooling2 (None, 7, 7, 32) 0 _________________________________________________________________ flatten (Flatten) (None, 1568) 0 _________________________________________________________________ dropout (Dropout) (None, 1568) 0 _________________________________________________________________ dense (Dense) (None, 64) 100416 _________________________________________________________________ dense_1 (Dense) (None, 4) 260 ================================================================= Total params: 495,140 Trainable params: 495,140 Non-trainable params: 0 _________________________________________________________________
# Train Model
history = model.fit(train_generator, epochs=10, validation_data=test_generator, shuffle=True, validation_steps=len(test_generator))
Epoch 1/10 49/49 [==============================] - 294s 6s/step - loss: 1.0938 - accuracy: 0.4851 - val_loss: 0.5189 - val_accuracy: 0.7740 Epoch 2/10 49/49 [==============================] - 256s 5s/step - loss: 0.4610 - accuracy: 0.8065 - val_loss: 0.3693 - val_accuracy: 0.8182 Epoch 3/10 49/49 [==============================] - 235s 5s/step - loss: 0.3532 - accuracy: 0.8474 - val_loss: 0.3160 - val_accuracy: 0.8519 Epoch 4/10 49/49 [==============================] - 233s 5s/step - loss: 0.3104 - accuracy: 0.8734 - val_loss: 0.3244 - val_accuracy: 0.8623 Epoch 5/10 49/49 [==============================] - 232s 5s/step - loss: 0.2882 - accuracy: 0.8870 - val_loss: 0.2186 - val_accuracy: 0.8987 Epoch 6/10 49/49 [==============================] - 229s 5s/step - loss: 0.2649 - accuracy: 0.9006 - val_loss: 0.1971 - val_accuracy: 0.9065 Epoch 7/10 49/49 [==============================] - 230s 5s/step - loss: 0.2359 - accuracy: 0.9045 - val_loss: 0.3169 - val_accuracy: 0.8545 Epoch 8/10 49/49 [==============================] - 229s 5s/step - loss: 0.2428 - accuracy: 0.8974 - val_loss: 0.1914 - val_accuracy: 0.9195 Epoch 9/10 49/49 [==============================] - 228s 5s/step - loss: 0.2114 - accuracy: 0.9110 - val_loss: 0.2047 - val_accuracy: 0.9065 Epoch 10/10 49/49 [==============================] - 228s 5s/step - loss: 0.2162 - accuracy: 0.9136 - val_loss: 0.1923 - val_accuracy: 0.9195
epochs = [i for i in range(10)]
fig , ax = plt.subplots(1,2)
train_acc = history.history['accuracy']
train_loss = history.history['loss']
test_acc = history.history['val_accuracy']
test_loss = history.history['val_loss']
fig.set_size_inches(20,6)
ax[0].plot(epochs , train_loss , label = 'Training Loss')
ax[0].plot(epochs , test_loss , label = 'Testing Loss')
ax[0].set_title('Training & Testing Loss')
ax[0].legend()
ax[0].set_xlabel("Epochs")
ax[1].plot(epochs , train_acc , label = 'Training Accuracy')
ax[1].plot(epochs , test_acc , label = 'Testing Accuracy')
ax[1].set_title('Training & Testing Accuracy')
ax[1].legend()
ax[1].set_xlabel("Epochs")
plt.show()
# Save Model
model.save("Trained_Model.h5")
# Predictions
y_pred = model.predict_classes(X_test)
y_test = [np.argmax(i) for i in y_test]
# Accuracy Score
accuracy = accuracy_score(y_test, y_pred)
print("\n\n======================")
print("Accuracy Score:")
print("======================\n\n")
print(" Accuracy: ", round(accuracy*100,2), "%")
# Classification Report
report = classification_report(y_test, y_pred)
print("\n\n======================")
print("Classification Report:")
print("======================\n\n")
print(report)
# Confusion Matrix
matrix = confusion_matrix(y_test, y_pred)
print("\n\n=================")
print("Confusion Matrix:")
print("=================\n\n")
print(matrix)
# Heat Map
print("\n\n===============================")
print("Confusion Matrix with Heat MAP:")
print("===============================\n\n")
confusion_matrix_heatmap = confusion_matrix(y_test, y_pred, normalize = 'true')
sns.set(rc={'figure.figsize':(10,7)})
sns.heatmap(confusion_matrix_heatmap, annot=True)
plt.show()
======================
Accuracy Score:
======================
Accuracy: 87.53 %
======================
Classification Report:
======================
precision recall f1-score support
0 0.95 0.43 0.60 46
1 0.68 0.98 0.80 52
2 0.93 0.89 0.91 127
3 0.92 0.96 0.94 160
accuracy 0.88 385
macro avg 0.87 0.82 0.81 385
weighted avg 0.89 0.88 0.87 385
=================
Confusion Matrix:
=================
[[ 20 24 2 0]
[ 0 51 1 0]
[ 0 0 113 14]
[ 1 0 6 153]]
===============================
Confusion Matrix with Heat MAP:
===============================
labels_new = ["Yawn", "No_Yawn", "Closed", "Open"]
IMG_SIZE = 145
def prepare(img, face_cas="Dataset/Face.xml"):
img_array = img
img_array = img_array / 255
resized_array = cv2.resize(img_array, (IMG_SIZE, IMG_SIZE))
return resized_array.reshape(-1, IMG_SIZE, IMG_SIZE, 3)
model = tf.keras.models.load_model("Trained_Model.h5")
# Get a reference to webcam
video_capture = cv2.VideoCapture(0)
while True:
ret, frame = video_capture.read()
frame = cv2.flip(frame, 1)
window_name = 'Live Detection'
font = cv2.FONT_HERSHEY_SIMPLEX
org = (50, 50)
fontScale = 2
color = (255, 0, 0)
thickness = 4
# Get Model Prediction
pred = model.predict([prepare(frame)])
pred = np.argmax(pred)
cv2.putText(frame, labels_new[pred], org, font, fontScale, color, thickness, cv2.LINE_AA)
cv2.imshow("Video", frame)
# Hit 'q' on the keyboard to quit!
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()