import numpy as np import tensorflow as tf import tensorflow_hub as hub import tf_keras as keras import pandas as pd from tensorflow.keras.models import load_model import classifier_data_lib import tokenization import joblib from deep_translator import GoogleTranslator import sys import os import gradio as gr model = load_model('ISCO-Coder-BERT.h5', custom_objects={'KerasLayer': hub.KerasLayer}) bert_layer = hub.KerasLayer("https://kaggle.com/models/tensorflow/bert/TensorFlow2/en-uncased-l-12-h-768-a-12/1",trainable=True) vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy() do_lower_case = bert_layer.resolved_object.do_lower_case.numpy() tokenizer = tokenization.FullTokenizer(vocab_file,do_lower_case) # Parameters max_seq_length = 128 dummy_label = 100 label_list = list(pd.read_excel('label_list.xlsx')['label_list']) # Define a function to preprocess the new data def get_feature_new(text, max_seq_length, tokenizer, dummy_label): example = classifier_data_lib.InputExample(guid=None, text_a=text.numpy().decode('utf-8'), text_b=None, label=dummy_label) # Use a valid dummy label feature = classifier_data_lib.convert_single_example(0, example, label_list, max_seq_length, tokenizer) return feature.input_ids, feature.input_mask, feature.segment_ids def get_feature_map_new(text): input_ids, input_mask, segment_ids = tf.py_function( lambda text: get_feature_new(text, max_seq_length, tokenizer, dummy_label), inp=[text], Tout=[tf.int32, tf.int32, tf.int32] ) input_ids.set_shape([max_seq_length]) input_mask.set_shape([max_seq_length]) segment_ids.set_shape([max_seq_length]) x = {'input_word_ids': input_ids, 'input_mask': input_mask, 'input_type_ids': segment_ids} return x def preprocess_new_data(texts): dataset = tf.data.Dataset.from_tensor_slices((texts,)) dataset = dataset.map(get_feature_map_new, num_parallel_calls=tf.data.experimental.AUTOTUNE) dataset = dataset.batch(32, drop_remainder=False) dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) return dataset def launch(text_input): # Load the label encoder label_encoder = joblib.load('label_encoder.joblib') # Preprocess the new data try: text_input = GoogleTranslator(source = 'auto',target = 'en').translate(text_input) except: text_input = text_input sample_example = [text_input] new_data_dataset = preprocess_new_data(sample_example) # Assuming you have a model already loaded (add model loading code if needed) # Make predictions on the new data predictions = model.predict(new_data_dataset) # Decode the predictions predicted_classes = [label_list[np.argmax(pred)] for pred in predictions] # Calculate the highest probabilities highest_probabilities = [max(instance) for instance in predictions] # Decode labels using the label encoder decoded_labels = label_encoder.inverse_transform(predicted_classes) print("Most likely ISCO code is {} and probability is {}".format(decoded_labels,highest_probabilities)) # Create descriptive text for the output result_text = "Most likely ISCO code is {} and probability is {:.2f}".format(decoded_labels[0], highest_probabilities[0]) print(result_text) iface = gr.Interface( fn=launch, inputs=gr.Textbox(lines=2, placeholder="Enter job title and description here..."), outputs="text" ) iface.launch()