from transformers import BertTokenizer
from datasets import load_dataset

# Load the dataset (assuming it's a CSV file)
dataset = load_dataset('csv', data_files='interview_data.csv', delimiter=',')

# Initialize tokenizer (using BERT tokenizer for example)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

# Tokenize the responses
def tokenize_function(examples):
    return tokenizer(examples['Response'], padding='max_length', truncation=True)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

# Split into train and test datasets
train_dataset = tokenized_datasets['train']
test_dataset = tokenized_datasets['test']