Update app.py

app.py

@@ -3,9 +3,7 @@ import numpy as np
 import pandas as pd
 from keras.models import load_model
 from huggingface_hub import hf_hub_download
-from tensorflow.keras.preprocessing.text import Tokenizer
-from tensorflow.keras.preprocessing.sequence import pad_sequences
-from tensorflow.keras.models import Sequential
+from transformers import BertTokenizer
 from sklearn.preprocessing import LabelEncoder, StandardScaler
 from nltk.sentiment.vader import SentimentIntensityAnalyzer
 import nltk
@@ -17,8 +15,10 @@ nltk.download('vader_lexicon')
 model_path = hf_hub_download(repo_id="xeroISB/ServiceNowMTTR", filename="my_model.h5")
 model = load_model(model_path)
 
-# Initialize Tokenizer and LabelEncoders
-tokenizer = Tokenizer(num_words=10000, oov_token='<OOV>')
+# Initialize BERT tokenizer
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+
+# Initialize LabelEncoders
 label_encoders = {
     'impact': LabelEncoder(),
     'priority': LabelEncoder(),
@@ -39,14 +39,12 @@ def preprocess_input(short_description, impact, priority, category, urgency):
     
     for column in ['impact', 'priority', 'category', 'urgency']:
         input_data[column] = label_encoders[column].fit_transform(input_data[column])
+    
     short_description = input_data['short_description'].iloc[0].lower()
-    # Tokenize text data
-    sequences = tokenizer.texts_to_sequences([short_description])
-    print("Short description",input_data['short_description'].iloc[0])
-    print("Sequence",sequences)
-    if not sequences:
-        return None, None  # Handle empty sequences
-    padded_sequences = pad_sequences(sequences, maxlen=50, padding='post', truncating='post')
+    
+    # Tokenize text data using BERT tokenizer
+    inputs = tokenizer(short_description, return_tensors='tf', padding='max_length', truncation=True, max_length=50)
+    padded_sequences = np.array(inputs['input_ids'])
     
     # Feature engineering: Add sentiment score
     sid = SentimentIntensityAnalyzer()
@@ -55,7 +53,6 @@ def preprocess_input(short_description, impact, priority, category, urgency):
     # Normalize numerical features
     numerical_features = input_data[['impact', 'priority', 'category', 'urgency', 'sentiment_score']]
     scaler = StandardScaler()
-    
     scaled_numerical_features = scaler.fit_transform(numerical_features)
 
     # Prepare the final input features