optimized the app

.gitignore

@@ -1 +1,3 @@
-venv
+venv
+sent_clf_onnx
+sentiment_model_dir

app.py

@@ -3,12 +3,17 @@ import pandas as pd
 import streamlit as st
 from streamlit_text_rating.st_text_rater import st_text_rater
 from sentiment import classify_sentiment
-from sentiment_onnx_classify import classify_sentiment_onnx, classify_sentiment_onnx_quant
+from sentiment_onnx_classify import classify_sentiment_onnx, classify_sentiment_onnx_quant,create_onnx_model
 from zeroshot_clf import zero_shot_classification
+from transformers import AutoTokenizer,AutoModelForSequenceClassification
+from onnxruntime.quantization import quantize_dynamic,QuantType
+import transformers.convert_graph_to_onnx as onnx_convert
+from pathlib import Path
+import os
 import time
 import plotly.express as px
 import plotly.graph_objects as go
-
+import onnxruntime as ort
 global _plotly_config
 _plotly_config={'displayModeBar': False}
 
@@ -66,6 +71,18 @@ hide_streamlit_style = """
 st.markdown(hide_streamlit_style, unsafe_allow_html=True)
 
 
+@st.cache(allow_output_mutation=True, suppress_st_warning=True, max_entries=None, ttl=None)
+def create_model_dir(chkpt, model_dir):
+    if not os.path.exists(chkpt):
+        os.mkdir(model_dir)
+        _model = AutoModelForSequenceClassification.from_pretrained(chkpt)
+        _tokenizer = AutoTokenizer.from_pretrained(chkpt)
+        _model.save_pretrained(model_dir)
+        _tokenizer.save_pretrained(model_dir)
+    else:
+        pass
+
+
 st.title("NLP use cases")
 
 with st.sidebar:
@@ -77,7 +94,36 @@ with st.sidebar:
 if select_task=='README':
     st.header("NLP Summary")
 
-if select_task=='Detect Sentiment':
+############### Pre-Download & instantiate objects for sentiment analysis *********************** START **********************
+sent_chkpt = 'distilbert-base-uncased-finetuned-sst-2-english'
+sent_model_dir='sentiment_model_dir'
+#create model/token dir
+create_model_dir(chkpt=sent_chkpt, model_dir=sent_model_dir)
+
+@st.cache(allow_output_mutation=True, suppress_st_warning=True, max_entries=None, ttl=None)
+def task_selected(task,sent_model_dir=sent_model_dir):
+    model_sentiment=AutoModelForSequenceClassification.from_pretrained(sent_model_dir)
+    tokenizer_sentiment=AutoTokenizer.from_pretrained(sent_model_dir)
+
+    create_onnx_model(_model=model_sentiment, _tokenizer=tokenizer_sentiment)
+
+    #create inference session
+    sentiment_session = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx.onnx")
+    sentiment_session_int8 = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx_int8.onnx")
+
+    return model_sentiment,tokenizer_sentiment,sentiment_session,sentiment_session_int8
+
+
+############## Pre-Download & instantiate objects for sentiment analysis ********************* END **********************************
+
+if select_task == 'Detect Sentiment':
+
+    t1=time.time()
+    model_sentiment,tokenizer_sentiment,\
+    sentiment_session,sentiment_session_int8 = task_selected(task=select_task)
+    t2 = time.time()
+    st.write(f"Total time to load Model is {(t2-t1)*1000:.1f} ms")
+
     st.header("You are now performing Sentiment Analysis")
     input_texts = st.text_input(label="Input texts separated by comma")
     c1,c2,c3,c4=st.columns(4)
@@ -94,33 +140,44 @@ if select_task=='Detect Sentiment':
     if any([response1,response2,response3,response4]):
         if response1:
             start=time.time()
-            sentiments = classify_sentiment(input_texts)
+            sentiments = classify_sentiment(input_texts,
+                                            model=model_sentiment,
+                                            tokenizer=tokenizer_sentiment
+                                            )
             end=time.time()
             st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
         elif response2:
             start = time.time()
-            sentiments=classify_sentiment_onnx(input_texts)
+            sentiments=classify_sentiment_onnx(input_texts,
+                                               _session=sentiment_session,
+                                               _tokenizer=tokenizer_sentiment)
             end = time.time()
             st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
         elif response3:
             start = time.time()
-            sentiments=classify_sentiment_onnx_quant(input_texts)
+            sentiments=classify_sentiment_onnx_quant(input_texts,
+                                                     _session=sentiment_session_int8,
+                                                     _tokenizer=tokenizer_sentiment)
             end = time.time()
             st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
         elif response4:
             normal_runtime=[]
             for i in range(100):
                 start=time.time()
-                sentiments = classify_sentiment(input_texts)
+                sentiments = classify_sentiment(input_texts,
+                                                model=model_sentiment,
+                                                tokenizer=tokenizer_sentiment)
                 end=time.time()
                 t = (end - start) * 1000
                 normal_runtime.append(t)
-            normal_runtime=np.clip(normal_runtime,10,40)
+            normal_runtime=np.clip(normal_runtime,10,60)
 
             onnx_runtime=[]
             for i in range(100):
                 start=time.time()
-                sentiments = classify_sentiment_onnx(input_texts)
+                sentiments = classify_sentiment_onnx(input_texts,
+                                               _session=sentiment_session,
+                                               _tokenizer=tokenizer_sentiment)
                 end=time.time()
                 t=(end-start)*1000
                 onnx_runtime.append(t)
@@ -129,12 +186,14 @@ if select_task=='Detect Sentiment':
             onnx_runtime_quant=[]
             for i in range(100):
                 start=time.time()
-                sentiments = classify_sentiment_onnx_quant(input_texts)
+                sentiments = classify_sentiment_onnx_quant(input_texts,
+                                               _session=sentiment_session,
+                                               _tokenizer=tokenizer_sentiment)
                 end=time.time()
 
                 t=(end-start)*1000
                 onnx_runtime_quant.append(t)
-            onnx_runtime_quant = np.clip(onnx_runtime_quant, 0, 10)
+            onnx_runtime_quant = np.clip(onnx_runtime_quant, 0, 20)
 
 
             temp_df=pd.DataFrame({'Normal Runtime (ms)':normal_runtime,
@@ -188,5 +247,3 @@ if select_task=='Zero Shot Classification':
             st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
             st.plotly_chart(output, config=_plotly_config)
 
-
-#awesome

sentiment.py

@@ -6,7 +6,7 @@ model=AutoModelForSequenceClassification.from_pretrained(chkpt)
 tokenizer=AutoTokenizer.from_pretrained(chkpt)
 # tokenizer=AutoTokenizer.from_pretrained('sentiment_classifier/')
 
-def classify_sentiment(texts,model=model,tokenizer=tokenizer):
+def classify_sentiment(texts,model,tokenizer):
     """
         user will pass texts separated by comma
     """

sentiment_classifier/config.json

@@ -1,34 +0,0 @@
-{
-  "_name_or_path": "distilbert-base-uncased-finetuned-sst-2-english",
-  "activation": "gelu",
-  "architectures": [
-    "DistilBertForSequenceClassification"
-  ],
-  "attention_dropout": 0.1,
-  "dim": 768,
-  "dropout": 0.1,
-  "finetuning_task": "sst-2",
-  "hidden_dim": 3072,
-  "id2label": {
-    "0": "NEGATIVE",
-    "1": "POSITIVE"
-  },
-  "initializer_range": 0.02,
-  "label2id": {
-    "NEGATIVE": 0,
-    "POSITIVE": 1
-  },
-  "max_position_embeddings": 512,
-  "model_type": "distilbert",
-  "n_heads": 12,
-  "n_layers": 6,
-  "output_past": true,
-  "pad_token_id": 0,
-  "qa_dropout": 0.1,
-  "seq_classif_dropout": 0.2,
-  "sinusoidal_pos_embds": false,
-  "tie_weights_": true,
-  "torch_dtype": "float32",
-  "transformers_version": "4.18.0",
-  "vocab_size": 30522
-}

sentiment_classifier/special_tokens_map.json

@@ -1 +0,0 @@
-{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

sentiment_classifier/tokenizer_config.json

@@ -1 +0,0 @@
-{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "distilbert-base-uncased-finetuned-sst-2-english", "do_basic_tokenize": true, "never_split": null, "tokenizer_class": "DistilBertTokenizer"}

sentiment_onnx.py

@@ -12,8 +12,7 @@ python3 -m transformers.onnx --model= distilbert-base-uncased-finetuned-sst-2-en
 """
 chkpt='distilbert-base-uncased-finetuned-sst-2-english'
 model= AutoModelForSequenceClassification.from_pretrained(chkpt)
-tokenizer= AutoTokenizer.from_pretrained('chkpt')
-# tokenizer= AutoTokenizer.from_pretrained('sentiment_classifier/')
+tokenizer= AutoTokenizer.from_pretrained(chkpt)
 
 """ 
 or download the model directly from hub --

sentiment_onnx_classify.py

@@ -1,23 +1,74 @@
 import onnxruntime as ort
 import torch
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer,AutoModelForSequenceClassification
 import numpy as np
+import transformers
+from onnxruntime.quantization import quantize_dynamic,QuantType
+import transformers.convert_graph_to_onnx as onnx_convert
+from pathlib import Path
+import os
 
 chkpt='distilbert-base-uncased-finetuned-sst-2-english'
-tokenizer=AutoTokenizer.from_pretrained(chkpt)
-# tokenizer=AutoTokenizer.from_pretrained("sentiment_classifier/")
+model= AutoModelForSequenceClassification.from_pretrained(chkpt)
+tokenizer= AutoTokenizer.from_pretrained(chkpt)
+
+def create_onnx_model(_model, _tokenizer):
+    """
+
+    Args:
+        _model: model checkpoint with AutoModelForSequenceClassification
+        _tokenizer: model checkpoint with AutoTokenizer
+
+    Returns:
+        Creates a simple ONNX model & int8 Quantized Model in the directory "sent_clf_onnx/" if directory not present
+
+    """
+    if not os.path.exists('sent_clf_onnx'):
+        os.mkdir('sent_clf_onnx')
+        """ 
+            Making ONNX model object
+        """
+        pipeline=transformers.pipeline("text-classification", model=_model, tokenizer=_tokenizer)
+
+        """ 
+            convert pipeline to onnx object
+        """
+        onnx_convert.convert_pytorch(pipeline,
+                                     opset=11,
+                                     output=Path("sent_clf_onnx/sentiment_classifier_onnx.onnx"),
+                                     use_external_format=False
+                                     )
+
+        """ 
+            convert onnx object to another onnx object with int8 quantization 
+        """
+        quantize_dynamic("sent_clf_onnx/sentiment_classifier_onnx.onnx","sent_clf_onnx/sentiment_classifier_onnx_int8.onnx",
+                         weight_type=QuantType.QUInt8)
+    else:
+        pass
+
+
+
 
 #create onnx & onnx_int_8 sessions
-session=ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx.onnx")
-session_int8=ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx_int8.onnx")
+session = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx.onnx")
+session_int8 = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx_int8.onnx")
 
 # options=ort.SessionOptions()
 # options.inter_op_num_threads=1
 # options.intra_op_num_threads=1
 
-def classify_sentiment_onnx(texts,_model=session,_tokenizer=tokenizer):
+def classify_sentiment_onnx(texts, _session, _tokenizer):
     """
-        user will pass texts separated by comma
+
+    Args:
+        texts: input texts from user
+        _session: pass ONNX runtime session
+        _tokenizer: Relevant Tokenizer e.g. AutoTokenizer.from_pretrained("same checkpoint as the model")
+
+    Returns:
+        list of Positve and Negative texts
+
     """
     try:
         texts=texts.split(',')
@@ -32,15 +83,22 @@ def classify_sentiment_onnx(texts,_model=session,_tokenizer=tokenizer):
         "attention_mask":np.array((_inputs['attention_mask']))
     }
 
-    output = _model.run(input_feed=input_feed, output_names=['output_0'])[0]
+    output = _session.run(input_feed=input_feed, output_names=['output_0'])[0]
 
     output=np.argmax(output,axis=1)
     output = ['Positive' if i == 1 else 'Negative' for i in output]
     return output
 
-def classify_sentiment_onnx_quant(texts, _model=session_int8, _tokenizer=tokenizer):
+def classify_sentiment_onnx_quant(texts, _session, _tokenizer):
     """
-        user will pass texts separated by comma
+    Args:
+        texts: input texts from user
+        _session: pass ONNX runtime session
+        _tokenizer: Relevant Tokenizer e.g. AutoTokenizer.from_pretrained("same checkpoint as the model")
+
+    Returns:
+        list of Positve and Negative texts
+
     """
     try:
         texts=texts.split(',')
@@ -56,7 +114,7 @@ def classify_sentiment_onnx_quant(texts, _model=session_int8, _tokenizer=tokeniz
         "attention_mask":np.array((_inputs['attention_mask']))
     }
 
-    output = _model.run(input_feed=input_feed, output_names=['output_0'])[0]
+    output = _session.run(input_feed=input_feed, output_names=['output_0'])[0]
 
     output=np.argmax(output,axis=1)
     output = ['Positive' if i == 1 else 'Negative' for i in output]