multilingual added

app.py

@@ -35,16 +35,10 @@ def get_sentences(further_cleaned_text):
     doc = nlp(further_cleaned_text)
     sentences = [sent.text for sent in doc.sents]
     return sentences
-    # doc = nlp(text)
-    # entities = []
-    # for ent in doc.ents:
-    #     if ent.label_ in labels:
-    #         entities.append((ent.text, ent.label_))
-    # return entities
-
-def get_model(model_name: str = None):
+
+def get_model(model_name: str = None, multilingual: bool = False):
     if model_name is None:
-        model_name = "urchade/gliner_base"
+        model_name = "urchade/gliner_base" if not multilingual else "urchade/gliner_multilingual"
 
     global _MODEL
 
@@ -53,11 +47,8 @@ def get_model(model_name: str = None):
 
     return _MODEL[model_name]
 
-def parse_query(sentences: List[str], labels: Union[str, list], threshold: float = 0.3, nested_ner: bool = False, model_name: str = None) -> List[Dict[str, Union[str, list]]]:
-    model = get_model(model_name)
-
-    if isinstance(labels, str):
-        labels = [i.strip() for i in labels.split(",")]
+def parse_query(sentences: List[str], labels: List[str], threshold: float = 0.3, nested_ner: bool = False, model_name: str = None, multilingual: bool = False) -> List[Dict[str, Union[str, list]]]:
+    model = get_model(model_name, multilingual=multilingual)
 
     results = []
 
@@ -73,14 +64,13 @@ def parse_query(sentences: List[str], labels: Union[str, list], threshold: float
 
     return results
 
-def present(email_content, labels):
+def present(email_content, labels, multilingual=False):
     email = accept_mail(email_content)
     cleaned_text = clean_email(email)
     further_cleaned_text = remove_special_characters(cleaned_text)
     sentence_list = get_sentences(further_cleaned_text)
-    # entity_info = '\n'.join([f"{text}: {label}" for text, label in entities])
     
-    result = parse_query(sentence_list, labels, threshold=0.3, nested_ner=False, model_name="urchade/gliner_base")
+    result = parse_query(sentence_list, labels, threshold=0.3, nested_ner=False, model_name="urchade/gliner_base", multilingual=multilingual)
 
     email_info = {
         "Subject": email.subject,
@@ -92,20 +82,18 @@ def present(email_content, labels):
     }
     return [email_info[key] for key in email_info]
 
-labels = ["PERSON", "PRODUCT", "DEAL", "ORDER", 
-          "ORDER PAYMENT METHOD", "STORE", "LEGAL ENTITY", 
-          "MERCHANT", "FINANCIAL TRANSACTION", "UNCATEGORIZED", "DATE"]
+labels = ["PERSON", "PRODUCT", "DEAL", "ORDER", "ORDER PAYMENT METHOD", "STORE", "LEGAL ENTITY", "MERCHANT", "FINANCIAL TRANSACTION", "UNCATEGORIZED", "DATE"]
 
 demo = gr.Interface(
     fn=present, 
     inputs=[
         gr.components.Textbox(label="Email Content"),
-        gr.components.Textbox(
-            value=", ".join(labels),
+        gr.components.CheckboxGroup(
+            choices=labels,
             label="Labels to Detect",
-            placeholder="Enter the entities to detect here (comma separated)",
-            scale=2
+            value=labels,  # Default all selected
         ),
+        gr.components.Checkbox(label="Use Multilingual Model")
     ],
     outputs=[
         gr.components.Textbox(label="Subject"),