# -*- coding: utf-8 -*-
"""MWP_Solver_-_Transformer_with_Multi-head_Attention_Block (1).ipynb
Automatically generated by Colaboratory.
Original file is located at
https://colab.research.google.com/drive/1Tn_j0k8EJ7ny_h7Pjm0stJhNMG4si_y_
"""
# ! pip install -q gradio
import pandas as pd
import re
import os
import time
import random
import numpy as np
os.system("pip install tensorflow")
os.system("pip install scikit-learn")
os.system("pip install spacy")
os.system("pip install nltk")
os.system("spacy download en_core_web_sm")
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from sklearn.model_selection import train_test_split
import pickle
import spacy
from nltk.translate.bleu_score import corpus_bleu
import gradio as gr
os.system("wget -nc 'https://docs.google.com/uc?export=download&id=1Y8Ee4lUs30BAfFtL3d3VjwChmbDG7O6H' -O data_final.pkl")
os.system('''wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1gAQVaxg_2mNcr8qwx0J2UwpkvoJgLu6a' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id=1gAQVaxg_2mNcr8qwx0J2UwpkvoJgLu6a" -O checkpoints.zip && rm -rf /tmp/cookies.txt''')
os.system("unzip -n './checkpoints.zip' -d './'")
nlp = spacy.load("en_core_web_sm")
tf.__version__
with open('data_final.pkl', 'rb') as f:
df = pickle.load(f)
df.shape
df.head()
input_exps = list(df['Question'].values)
def convert_eqn(eqn):
'''
Add a space between every character in the equation string.
Eg: 'x = 23 + 88' becomes 'x = 2 3 + 8 8'
'''
elements = list(eqn)
return ' '.join(elements)
target_exps = list(df['Equation'].apply(lambda x: convert_eqn(x)).values)
# Input: Word problem
input_exps[:5]
# Target: Equation
target_exps[:5]
len(pd.Series(input_exps)), len(pd.Series(input_exps).unique())
len(pd.Series(target_exps)), len(pd.Series(target_exps).unique())
def preprocess_input(sentence):
'''
For the word problem, convert everything to lowercase, add spaces around all
punctuations and digits, and remove any extra spaces.
'''
sentence = sentence.lower().strip()
sentence = re.sub(r"([?.!,’])", r" \1 ", sentence)
sentence = re.sub(r"([0-9])", r" \1 ", sentence)
sentence = re.sub(r'[" "]+', " ", sentence)
sentence = sentence.rstrip().strip()
return sentence
def preprocess_target(sentence):
'''
For the equation, convert it to lowercase and remove extra spaces
'''
sentence = sentence.lower().strip()
return sentence
preprocessed_input_exps = list(map(preprocess_input, input_exps))
preprocessed_target_exps = list(map(preprocess_target, target_exps))
preprocessed_input_exps[:5]
preprocessed_target_exps[:5]
def tokenize(lang):
'''
Tokenize the given list of strings and return the tokenized output
along with the fitted tokenizer.
'''
lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
lang_tokenizer.fit_on_texts(lang)
tensor = lang_tokenizer.texts_to_sequences(lang)
return tensor, lang_tokenizer
input_tensor, inp_lang_tokenizer = tokenize(preprocessed_input_exps)
len(inp_lang_tokenizer.word_index)
target_tensor, targ_lang_tokenizer = tokenize(preprocessed_target_exps)
old_len = len(targ_lang_tokenizer.word_index)
def append_start_end(x,last_int):
'''
Add integers for start and end tokens for input/target exps
'''
l = []
l.append(last_int+1)
l.extend(x)
l.append(last_int+2)
return l
input_tensor_list = [append_start_end(i,len(inp_lang_tokenizer.word_index)) for i in input_tensor]
target_tensor_list = [append_start_end(i,len(targ_lang_tokenizer.word_index)) for i in target_tensor]
# Pad all sequences such that they are of equal length
input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor_list, padding='post')
target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor_list, padding='post')
input_tensor
target_tensor
# Here we are increasing the vocabulary size of the target, by adding a
# few extra vocabulary words (which will not actually be used) as otherwise the
# small vocab size causes issues downstream in the network.
keys = [str(i) for i in range(10,51)]
for i,k in enumerate(keys):
targ_lang_tokenizer.word_index[k]=len(targ_lang_tokenizer.word_index)+i+4
len(targ_lang_tokenizer.word_index)
# Creating training and validation sets
input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor,
target_tensor,
test_size=0.05,
random_state=42)
len(input_tensor_train)
len(input_tensor_val)
BUFFER_SIZE = len(input_tensor_train)
BATCH_SIZE = 64
steps_per_epoch = len(input_tensor_train)//BATCH_SIZE
dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)
dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)
num_layers = 4
d_model = 128
dff = 512
num_heads = 8
input_vocab_size = len(inp_lang_tokenizer.word_index)+3
target_vocab_size = len(targ_lang_tokenizer.word_index)+3
dropout_rate = 0.0
example_input_batch, example_target_batch = next(iter(dataset))
example_input_batch.shape, example_target_batch.shape
# We provide positional information about the data to the model,
# otherwise each sentence will be treated as Bag of Words
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))
return pos * angle_rates
def positional_encoding(position, d_model):
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
# apply sin to even indices in the array; 2i
angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
# apply cos to odd indices in the array; 2i+1
angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
pos_encoding = angle_rads[np.newaxis, ...]
return tf.cast(pos_encoding, dtype=tf.float32)
# mask all elements are that not words (padding) so that it is not treated as input
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
# add extra dimensions to add the padding
# to the attention logits.
return seq[:, tf.newaxis, tf.newaxis, :] # (batch_size, 1, 1, seq_len)
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
return mask
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
def scaled_dot_product_attention(q, k, v, mask):
matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k)
# scale matmul_qk
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
# add the mask to the scaled tensor.
if mask is not None:
scaled_attention_logits += (mask * -1e9)
# softmax is normalized on the last axis (seq_len_k) so that the scores
# add up to 1.
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # (..., seq_len_q, seq_len_k)
output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v)
return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
self.wk = tf.keras.layers.Dense(d_model)
self.wv = tf.keras.layers.Dense(d_model)
self.dense = tf.keras.layers.Dense(d_model)
def split_heads(self, x, batch_size):
"""Split the last dimension into (num_heads, depth).
Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)
"""
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q) # (batch_size, seq_len, d_model)
k = self.wk(k) # (batch_size, seq_len, d_model)
v = self.wv(v) # (batch_size, seq_len, d_model)
q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)
k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth)
v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth)
# scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
# attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
scaled_attention, attention_weights = scaled_dot_product_attention(
q, k, v, mask)
scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) # (batch_size, seq_len_q, num_heads, depth)
concat_attention = tf.reshape(scaled_attention,
(batch_size, -1, self.d_model)) # (batch_size, seq_len_q, d_model)
output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model)
return output, attention_weights
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'), # (batch_size, seq_len, dff)
tf.keras.layers.Dense(d_model) # (batch_size, seq_len, d_model)
])
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
# normalize data per feature instead of batch
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
# Multi-head attention layer
attn_output, _ = self.mha(x, x, x, mask)
attn_output = self.dropout1(attn_output, training=training)
# add residual connection to avoid vanishing gradient problem
out1 = self.layernorm1(x + attn_output)
# Feedforward layer
ffn_output = self.ffn(out1)
ffn_output = self.dropout2(ffn_output, training=training)
# add residual connection to avoid vanishing gradient problem
out2 = self.layernorm2(out1 + ffn_output)
return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
maximum_position_encoding, rate=0.1):
super(Encoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
# Create encoder layers (count: num_layers)
self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
seq_len = tf.shape(x)[1]
# adding embedding and position encoding.
x = self.embedding(x)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
return x
class DecoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(DecoderLayer, self).__init__()
self.mha1 = MultiHeadAttention(d_model, num_heads)
self.mha2 = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.dropout3 = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
# Masked multihead attention layer (padding + look-ahead)
attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)
attn1 = self.dropout1(attn1, training=training)
# again add residual connection
out1 = self.layernorm1(attn1 + x)
# Masked multihead attention layer (only padding)
# with input from encoder as Key and Value, and input from previous layer as Query
attn2, attn_weights_block2 = self.mha2(
enc_output, enc_output, out1, padding_mask)
attn2 = self.dropout2(attn2, training=training)
# again add residual connection
out2 = self.layernorm2(attn2 + out1)
# Feedforward layer
ffn_output = self.ffn(out2)
ffn_output = self.dropout3(ffn_output, training=training)
# again add residual connection
out3 = self.layernorm3(ffn_output + out2)
return out3, attn_weights_block1, attn_weights_block2
class Decoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,
maximum_position_encoding, rate=0.1):
super(Decoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
# Create decoder layers (count: num_layers)
self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
seq_len = tf.shape(x)[1]
attention_weights = {}
x = self.embedding(x) # (batch_size, target_seq_len, d_model)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:,:seq_len,:]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x, block1, block2 = self.dec_layers[i](x, enc_output, training,
look_ahead_mask, padding_mask)
# store attenion weights, they can be used to visualize while translating
attention_weights['decoder_layer{}_block1'.format(i+1)] = block1
attention_weights['decoder_layer{}_block2'.format(i+1)] = block2
return x, attention_weights
class Transformer(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
target_vocab_size, pe_input, pe_target, rate=0.1):
super(Transformer, self).__init__()
self.encoder = Encoder(num_layers, d_model, num_heads, dff,
input_vocab_size, pe_input, rate)
self.decoder = Decoder(num_layers, d_model, num_heads, dff,
target_vocab_size, pe_target, rate)
self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def call(self, inp, tar, training, enc_padding_mask,
look_ahead_mask, dec_padding_mask):
# Pass the input to the encoder
enc_output = self.encoder(inp, training, enc_padding_mask)
# Pass the encoder output to the decoder
dec_output, attention_weights = self.decoder(
tar, enc_output, training, look_ahead_mask, dec_padding_mask)
# Pass the decoder output to the last linear layer
final_output = self.final_layer(dec_output)
return final_output, attention_weights
class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, d_model, warmup_steps=4000):
super(CustomSchedule, self).__init__()
self.d_model = d_model
self.d_model = tf.cast(self.d_model, tf.float32)
self.warmup_steps = warmup_steps
def __call__(self, step):
arg1 = tf.math.rsqrt(step)
arg2 = step * (self.warmup_steps ** -1.5)
return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
learning_rate = CustomSchedule(d_model)
# Adam optimizer with a custom learning rate
optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,
epsilon=1e-9)
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True, reduction='none')
def loss_function(real, pred):
# Apply a mask to paddings (0)
mask = tf.math.logical_not(tf.math.equal(real, 0))
loss_ = loss_object(real, pred)
mask = tf.cast(mask, dtype=loss_.dtype)
loss_ *= mask
return tf.reduce_mean(loss_)
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
name='train_accuracy')
transformer = Transformer(num_layers, d_model, num_heads, dff,
input_vocab_size, target_vocab_size,
pe_input=input_vocab_size,
pe_target=target_vocab_size,
rate=dropout_rate)
def create_masks(inp, tar):
# Encoder padding mask
enc_padding_mask = create_padding_mask(inp)
# Decoder padding mask
dec_padding_mask = create_padding_mask(inp)
# Look ahead mask (for hiding the rest of the sequence in the 1st decoder attention layer)
look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])
dec_target_padding_mask = create_padding_mask(tar)
combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)
return enc_padding_mask, combined_mask, dec_padding_mask
# drive_root = '/gdrive/My Drive/'
drive_root = './'
checkpoint_dir = os.path.join(drive_root, "checkpoints")
checkpoint_dir = os.path.join(checkpoint_dir, "training_checkpoints/moops_transfomer")
print("Checkpoints directory is", checkpoint_dir)
if os.path.exists(checkpoint_dir):
print("Checkpoints folder already exists")
else:
print("Creating a checkpoints directory")
os.makedirs(checkpoint_dir)
checkpoint = tf.train.Checkpoint(transformer=transformer,
optimizer=optimizer)
ckpt_manager = tf.train.CheckpointManager(checkpoint, checkpoint_dir, max_to_keep=5)
latest = ckpt_manager.latest_checkpoint
latest
if latest:
epoch_num = int(latest.split('/')[-1].split('-')[-1])
checkpoint.restore(latest)
print ('Latest checkpoint restored!!')
else:
epoch_num = 0
epoch_num
# EPOCHS = 17
# def train_step(inp, tar):
# tar_inp = tar[:, :-1]
# tar_real = tar[:, 1:]
# enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)
# with tf.GradientTape() as tape:
# predictions, _ = transformer(inp, tar_inp,
# True,
# enc_padding_mask,
# combined_mask,
# dec_padding_mask)
# loss = loss_function(tar_real, predictions)
# gradients = tape.gradient(loss, transformer.trainable_variables)
# optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))
# train_loss(loss)
# train_accuracy(tar_real, predictions)
# for epoch in range(epoch_num, EPOCHS):
# start = time.time()
# train_loss.reset_states()
# train_accuracy.reset_states()
# # inp -> question, tar -> equation
# for (batch, (inp, tar)) in enumerate(dataset):
# train_step(inp, tar)
# if batch % 50 == 0:
# print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(
# epoch + 1, batch, train_loss.result(), train_accuracy.result()))
# ckpt_save_path = ckpt_manager.save()
# print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,
# ckpt_save_path))
# print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1,
# train_loss.result(),
# train_accuracy.result()))
# print ('Time taken for 1 epoch: {} secs\n'.format(time.time() - start))
def evaluate(inp_sentence):
start_token = [len(inp_lang_tokenizer.word_index)+1]
end_token = [len(inp_lang_tokenizer.word_index)+2]
# inp sentence is the word problem, hence adding the start and end token
inp_sentence = start_token + [inp_lang_tokenizer.word_index.get(i, inp_lang_tokenizer.word_index['john']) for i in preprocess_input(inp_sentence).split(' ')] + end_token
encoder_input = tf.expand_dims(inp_sentence, 0)
# start with equation's start token
decoder_input = [old_len+1]
output = tf.expand_dims(decoder_input, 0)
for i in range(MAX_LENGTH):
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
encoder_input, output)
predictions, attention_weights = transformer(encoder_input,
output,
False,
enc_padding_mask,
combined_mask,
dec_padding_mask)
# select the last word from the seq_len dimension
predictions = predictions[: ,-1:, :]
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
# return the result if the predicted_id is equal to the end token
if predicted_id == old_len+2:
return tf.squeeze(output, axis=0), attention_weights
# concatentate the predicted_id to the output which is given to the decoder
# as its input.
output = tf.concat([output, predicted_id], axis=-1)
return tf.squeeze(output, axis=0), attention_weights
# def plot_attention_weights(attention, sentence, result, layer):
# fig = plt.figure(figsize=(16, 8))
# sentence = preprocess_input(sentence)
# attention = tf.squeeze(attention[layer], axis=0)
# for head in range(attention.shape[0]):
# ax = fig.add_subplot(2, 4, head+1)
# # plot the attention weights
# ax.matshow(attention[head][:-1, :], cmap='viridis')
# fontdict = {'fontsize': 10}
# ax.set_xticks(range(len(sentence.split(' '))+2))
# ax.set_yticks(range(len([targ_lang_tokenizer.index_word[i] for i in list(result.numpy())
# if i < len(targ_lang_tokenizer.word_index) and i not in [0,old_len+1,old_len+2]])+3))
# ax.set_ylim(len([targ_lang_tokenizer.index_word[i] for i in list(result.numpy())
# if i < len(targ_lang_tokenizer.word_index) and i not in [0,old_len+1,old_len+2]]), -0.5)
# ax.set_xticklabels(
# ['<start>']+sentence.split(' ')+['<end>'],
# fontdict=fontdict, rotation=90)
# ax.set_yticklabels([targ_lang_tokenizer.index_word[i] for i in list(result.numpy())
# if i < len(targ_lang_tokenizer.word_index) and i not in [0,old_len+1,old_len+2]],
# fontdict=fontdict)
# ax.set_xlabel('Head {}'.format(head+1))
# plt.tight_layout()
# plt.show()
MAX_LENGTH = 40
def translate(sentence, plot=''):
result, attention_weights = evaluate(sentence)
# use the result tokens to convert prediction into a list of characters
# (not inclusing padding, start and end tokens)
predicted_sentence = [targ_lang_tokenizer.index_word[i] for i in list(result.numpy()) if (i < len(targ_lang_tokenizer.word_index) and i not in [0,46,47])]
# print('Input: {}'.format(sentence))
return ''.join(predicted_sentence)
if plot:
plot_attention_weights(attention_weights, sentence, result, plot)
# def evaluate_results(inp_sentence):
# start_token = [len(inp_lang_tokenizer.word_index)+1]
# end_token = [len(inp_lang_tokenizer.word_index)+2]
# # inp sentence is the word problem, hence adding the start and end token
# inp_sentence = start_token + list(inp_sentence.numpy()[0]) + end_token
# encoder_input = tf.expand_dims(inp_sentence, 0)
# decoder_input = [old_len+1]
# output = tf.expand_dims(decoder_input, 0)
# for i in range(MAX_LENGTH):
# enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
# encoder_input, output)
# # predictions.shape == (batch_size, seq_len, vocab_size)
# predictions, attention_weights = transformer(encoder_input,
# output,
# False,
# enc_padding_mask,
# combined_mask,
# dec_padding_mask)
# # select the last word from the seq_len dimension
# predictions = predictions[: ,-1:, :] # (batch_size, 1, vocab_size)
# predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
# # return the result if the predicted_id is equal to the end token
# if predicted_id == old_len+2:
# return tf.squeeze(output, axis=0), attention_weights
# # concatentate the predicted_id to the output which is given to the decoder
# # as its input.
# output = tf.concat([output, predicted_id], axis=-1)
# return tf.squeeze(output, axis=0), attention_weights
# dataset_val = tf.data.Dataset.from_tensor_slices((input_tensor_val, target_tensor_val)).shuffle(BUFFER_SIZE)
# dataset_val = dataset_val.batch(1, drop_remainder=True)
# y_true = []
# y_pred = []
# acc_cnt = 0
# a = 0
# for (inp_val_batch, target_val_batch) in iter(dataset_val):
# a += 1
# if a % 100 == 0:
# print(a)
# print("Accuracy count: ",acc_cnt)
# print('------------------')
# target_sentence = ''
# for i in target_val_batch.numpy()[0]:
# if i not in [0,old_len+1,old_len+2]:
# target_sentence += (targ_lang_tokenizer.index_word[i] + ' ')
# y_true.append([target_sentence.split(' ')[:-1]])
# result, _ = evaluate_results(inp_val_batch)
# predicted_sentence = [targ_lang_tokenizer.index_word[i] for i in list(result.numpy()) if (i < len(targ_lang_tokenizer.word_index) and i not in [0,old_len+1,old_len+2])]
# y_pred.append(predicted_sentence)
# if target_sentence.split(' ')[:-1] == predicted_sentence:
# acc_cnt += 1
# len(y_true), len(y_pred)
# print('Corpus BLEU score of the model: ', corpus_bleu(y_true, y_pred))
# print('Accuracy of the model: ', acc_cnt/len(input_tensor_val))
check_str = ' '.join([inp_lang_tokenizer.index_word[i] for i in input_tensor_val[242] if i not in [0,
len(inp_lang_tokenizer.word_index)+1,
len(inp_lang_tokenizer.word_index)+2]])
check_str
translate(check_str)
#'victor had some car . john took 3 0 from him . now victor has 6 8 car . how many car victor had originally ?'
translate('Nafis had 31 raspberry . He slice each raspberry into 19 slices . How many raspberry slices did Denise make?')
interface = gr.Interface(
fn = translate,
inputs = gr.inputs.Textbox(lines = 2),
outputs = 'text',
examples = [
['Rachel bought two coloring books. One had 23 pictures and the other had 32. After one week she had colored 19 of the pictures. How many pictures does she still have to color?'],
['Denise had 31 raspberries. He slices each raspberry into 19 slices. How many raspberry slices did Denise make?'],
['A painter needed to paint 12 rooms in a building. Each room takes 7 hours to paint. If he already painted 5 rooms, how much longer will he take to paint the rest?'],
['Jerry had 135 pens. John took 19 pens from him. How many pens Jerry have left?'],
['Donald had some apples. Hillary took 20 apples from him. Now Donald has 100 apples. How many apples Donald had before?']
],
title = 'Mathbot',
description = 'Enter a simple math word problem and our AI will try to predict an expression to solve it. Mathbot occasionally makes mistakes. Feel free to press "flag" if you encounter such a scenario.',
)
interface.launch()