Spaces:
Sleeping
Sleeping
File size: 3,577 Bytes
4d7f8fd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
import re
import tensorflow as tf
import tensorflow_datasets as tfds
import nltk
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')
nltk.download('punkt')
lemmatizer = WordNetLemmatizer()
def preprocess_sentence(sentence):
sentence = sentence.lower().strip()
sentence = re.sub(r"([?.!¿])", r" \1 ", sentence)
sentence = re.sub(r'[" "]+', " ", sentence)
sentence = re.sub(r"[-()\"#/@;:<>{}+=~|.?,]", "", sentence)
sentence = re.sub(r"[^a-zA-ZğüşöçıİĞÜŞÖÇ?.!,¿]+", " ", sentence)
sentence = sentence.strip()
sentence = ' '.join([lemmatizer.lemmatize(w) for w in nltk.word_tokenize(sentence)])
return sentence
def load_conversations(hparams, lines_file, conversations_file):
id2line = {}
with open(lines_file, encoding = "utf-8", errors="ignore") as file:
lines = file.readlines()
for line in lines:
parts = line.replace("\n", "").split(" +++$+++ ")
id2line[parts[0]] = parts[4]
questions = []
answers = []
with open(conversations_file, "r") as file:
lines = file.readlines()
for line in lines:
parts = line.replace("\n", "").split(" +++$+++ ")
conversation = [line[1:-1] for line in parts[3][1:-1].split(", ")]
for i in range(len(conversation) - 1):
questions.append(preprocess_sentence(id2line[conversation[i]]))
answers.append(preprocess_sentence(id2line[conversation[i + 1]]))
if len(questions) >= hparams.max_samples:
return questions, answers
return questions, answers
def tokenize(hparams, tokenizer, questions, answers):
tokenized_inputs, tokenized_outputs = [], []
for (question, answer) in zip(questions, answers):
sentence1 = hparams.start_token + tokenizer.encode(question) + hparams.end_token
sentence2 = hparams.start_token + tokenizer.encode(answer) + hparams.end_token
if (len(sentence1) <= hparams.max_length and len(sentence2) <= hparams.max_length):
tokenized_inputs.append(sentence1)
tokenized_outputs.append(sentence2)
tokenized_inputs = tf.keras.preprocessing.sequence.pad_sequences(
tokenized_inputs, maxlen=hparams.max_length, padding="post")
tokenized_outputs = tf.keras.preprocessing.sequence.pad_sequences(
tokenized_outputs, maxlen=hparams.max_length, padding="post")
return tokenized_inputs, tokenized_outputs
def get_dataset(hparams):
lines_file ="data/lines.txt"
conversations_file = "data/conversations.txt"
questions, answers = load_conversations(hparams, lines_file, conversations_file)
tokenizer = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(questions + answers, target_vocab_size=2**13)
tokenizer.save_to_file('tokenizer')
hparams.start_token = [tokenizer.vocab_size]
hparams.end_token = [tokenizer.vocab_size + 1]
hparams.vocab_size = tokenizer.vocab_size + 2
questions, answers = tokenize(hparams, tokenizer, questions, answers)
dataset = tf.data.Dataset.from_tensor_slices(
({"inputs": questions, "dec_inputs": answers[:, :-1]}, answers[:, 1:])
)
dataset = dataset.cache()
dataset = dataset.shuffle(len(questions))
dataset = dataset.batch(hparams.batch_size)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
return dataset, tokenizer
|