-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy path3. Training the SEQ2SEQ model.py
135 lines (122 loc) · 8.17 KB
/
3. Training the SEQ2SEQ model.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
########## PART 3 - TRAINING THE SEQ2SEQ MODEL ##########
# Setting the Hyperparameters
epochs = 100
batch_size = 32
rnn_size = 1024
num_layers = 3
encoding_embedding_size = 1024
decoding_embedding_size = 1024
learning_rate = 0.001
learning_rate_decay = 0.9
min_learning_rate = 0.0001
keep_probability = 0.5
# Defining a session
tf.reset_default_graph()
session = tf.InteractiveSession()
# Loading the model inputs
inputs, targets, lr, keep_prob = model_inputs()
# Setting the sequence length
sequence_length = tf.placeholder_with_default(25, None, name = 'sequence_length')
# Getting the shape of the inputs tensor
input_shape = tf.shape(inputs)
# Getting the training and test predictions
training_predictions, test_predictions = seq2seq_model(tf.reverse(inputs, [-1]),
targets,
keep_prob,
batch_size,
sequence_length,
len(answerswords2int),
len(questionswords2int),
encoding_embedding_size,
decoding_embedding_size,
rnn_size,
num_layers,
questionswords2int)
# Setting up the Loss Error, the Optimizer and Gradient Clipping
with tf.name_scope("optimization"):
loss_error = tf.contrib.seq2seq.sequence_loss(training_predictions,
targets,
tf.ones([input_shape[0], sequence_length]))
optimizer = tf.train.AdamOptimizer(learning_rate)
gradients = optimizer.compute_gradients(loss_error)
clipped_gradients = [(tf.clip_by_value(grad_tensor, -5., 5.), grad_variable) for grad_tensor, grad_variable in gradients if grad_tensor is not None]
optimizer_gradient_clipping = optimizer.apply_gradients(clipped_gradients)
# Padding the sequences with the <PAD> token
def apply_padding(batch_of_sequences, word2int):
max_sequence_length = max([len(sequence) for sequence in batch_of_sequences])
return [sequence + [word2int['<PAD>']] * (max_sequence_length - len(sequence)) for sequence in batch_of_sequences]
# Splitting the data into batches of questions and answers
def split_into_batches(questions, answers, batch_size):
for batch_index in range(0, len(questions) // batch_size):
start_index = batch_index * batch_size
questions_in_batch = questions[start_index : start_index + batch_size]
answers_in_batch = answers[start_index : start_index + batch_size]
padded_questions_in_batch = np.array(apply_padding(questions_in_batch, questionswords2int))
padded_answers_in_batch = np.array(apply_padding(answers_in_batch, answerswords2int))
yield padded_questions_in_batch, padded_answers_in_batch
# Splitting the questions and answers into training and validation sets
training_validation_split = int(len(sorted_clean_questions) * 0.15)
training_questions = sorted_clean_questions[training_validation_split:]
training_answers = sorted_clean_answers[training_validation_split:]
validation_questions = sorted_clean_questions[:training_validation_split]
validation_answers = sorted_clean_answers[:training_validation_split]
# Training
batch_index_check_training_loss = 100
batch_index_check_validation_loss = ((len(training_questions)) // batch_size // 2) - 1
total_training_loss_error = 0
list_validation_loss_error = []
early_stopping_check = 0
early_stopping_stop = 100
checkpoint = "chatbot_weights.ckpt"
session.run(tf.global_variables_initializer())
for epoch in range(1, epochs + 1):
for batch_index, (padded_questions_in_batch, padded_answers_in_batch) in enumerate(split_into_batches(training_questions, training_answers, batch_size)):
starting_time = time.time()
_, batch_training_loss_error = session.run([optimizer_gradient_clipping, loss_error], {inputs: padded_questions_in_batch,
targets: padded_answers_in_batch,
lr: learning_rate,
sequence_length: padded_answers_in_batch.shape[1],
keep_prob: keep_probability})
total_training_loss_error += batch_training_loss_error
ending_time = time.time()
batch_time = ending_time - starting_time
if batch_index % batch_index_check_training_loss == 0:
print('Epoch: {:>3}/{}, Batch: {:>4}/{}, Training Loss Error: {:>6.3f}, Training Time on 100 Batches: {:d} seconds'.format(epoch,
epochs,
batch_index,
len(training_questions) // batch_size,
total_training_loss_error / batch_index_check_training_loss,
int(batch_time * batch_index_check_training_loss)))
total_training_loss_error = 0
if batch_index % batch_index_check_validation_loss == 0 and batch_index > 0:
total_validation_loss_error = 0
starting_time = time.time()
for batch_index_validation, (padded_questions_in_batch, padded_answers_in_batch) in enumerate(split_into_batches(validation_questions, validation_answers, batch_size)):
batch_validation_loss_error = session.run(loss_error, {inputs: padded_questions_in_batch,
targets: padded_answers_in_batch,
lr: learning_rate,
sequence_length: padded_answers_in_batch.shape[1],
keep_prob: 1})
total_validation_loss_error += batch_validation_loss_error
ending_time = time.time()
batch_time = ending_time - starting_time
average_validation_loss_error = total_validation_loss_error / (len(validation_questions) / batch_size)
print('Validation Loss Error: {:>6.3f}, Batch Validation Time: {:d} seconds'.format(average_validation_loss_error, int(batch_time)))
learning_rate *= learning_rate_decay
if learning_rate < min_learning_rate:
learning_rate = min_learning_rate
list_validation_loss_error.append(average_validation_loss_error)
if average_validation_loss_error <= min(list_validation_loss_error):
print('I speak better now!!')
early_stopping_check = 0
saver = tf.train.Saver()
saver.save(session, checkpoint)
else:
print("Sorry I do not speak better, I need to practice more.")
early_stopping_check += 1
if early_stopping_check == early_stopping_stop:
break
if early_stopping_check == early_stopping_stop:
print("My apologies, I cannot speak better anymore. This is the best I can do.")
break
print("Game Over")