Upload 4 files

README.txt

@@ -1,10 +1,10 @@
 ---
-title: 🗣️SpeakNowASR🧠Memory💾Gradio
-emoji: 🗣️🧠💾
-colorFrom: yellow
+title: ⚕️MedNER - Biomed Entity Recognizer
+emoji: 👩‍⚕️🩺⚕️🙋
+colorFrom: purple
 colorTo: red
 sdk: gradio
-sdk_version: 3.5
+sdk_version: 3.8
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -1,168 +1,81 @@
 import gradio as gr
-import torch
-import time
-import librosa
-import soundfile
-import nemo.collections.asr as nemo_asr
-import tempfile
-import os
-import uuid
-
-from transformers import BlenderbotTokenizer, BlenderbotForConditionalGeneration
-import torch
-
-# PersistDataset -----
-import os
-import csv
-import gradio as gr
-from gradio import inputs, outputs
-import huggingface_hub
-from huggingface_hub import Repository, hf_hub_download, upload_file
-from datetime import datetime
-
-# ---------------------------------------------
-# Dataset and Token links - change awacke1 to your own HF id, and add a HF_TOKEN copy to your repo for write permissions
-# This should allow you to save your results to your own Dataset hosted on HF. ---
-#DATASET_REPO_URL = "https://huggingface.co/datasets/awacke1/Carddata.csv"
-#DATASET_REPO_ID = "awacke1/Carddata.csv"
-#DATA_FILENAME = "Carddata.csv"
-#DATA_FILE = os.path.join("data", DATA_FILENAME)
-#HF_TOKEN = os.environ.get("HF_TOKEN")
-#SCRIPT = """
-
-#<script>
-#if (!window.hasBeenRun) {
-#    window.hasBeenRun = true;
-#    console.log("should only happen once");
-#    document.querySelector("button.submit").click();
-#}
-#</script>
-#"""
-
-#try:
-#    hf_hub_download(
-#        repo_id=DATASET_REPO_ID,
-#        filename=DATA_FILENAME,
-#        cache_dir=DATA_DIRNAME,
-#        force_filename=DATA_FILENAME
-#    )
-#except:
-#    print("file not found")
-#repo = Repository(
-#    local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN
-#)
-           
-#def store_message(name: str, message: str):
-#    if name and message:
-#        with open(DATA_FILE, "a") as csvfile:
-#            writer = csv.DictWriter(csvfile, fieldnames=["name", "message", "time"])
-#            writer.writerow(
-#                {"name": name.strip(), "message": message.strip(), "time": str(datetime.now())}
-#            )
-#        # uncomment line below to begin saving - 
-#        commit_url = repo.push_to_hub()
-#    return ""            
-
-#iface = gr.Interface(
-#    store_message,
-#    [
-#        inputs.Textbox(placeholder="Your name"),
-#        inputs.Textbox(placeholder="Your message", lines=2),
-#    ],
-#    "html",
-#    css="""
-#    .message {background-color:cornflowerblue;color:white; padding:4px;margin:4px;border-radius:4px; }
-#    """,
-#    title="Reading/writing to a HuggingFace dataset repo from Spaces",
-#    description=f"This is a demo of how to do simple *shared data persistence* in a Gradio Space, backed by a dataset repo.",
-#    article=f"The dataset repo is [{DATASET_REPO_URL}]({DATASET_REPO_URL})",
-#)
-
-
-# main -------------------------
-mname = "facebook/blenderbot-400M-distill"
-model = BlenderbotForConditionalGeneration.from_pretrained(mname)
-tokenizer = BlenderbotTokenizer.from_pretrained(mname)
-
-def take_last_tokens(inputs, note_history, history):
-    """Filter the last 128 tokens"""
-    if inputs['input_ids'].shape[1] > 128:
-        inputs['input_ids'] = torch.tensor([inputs['input_ids'][0][-128:].tolist()])
-        inputs['attention_mask'] = torch.tensor([inputs['attention_mask'][0][-128:].tolist()])
-        note_history = ['</s> <s>'.join(note_history[0].split('</s> <s>')[2:])]
-        history = history[1:]
-    return inputs, note_history, history
-
-def add_note_to_history(note, note_history):
-    """Add a note to the historical information"""
-    note_history.append(note)
-    note_history = '</s> <s>'.join(note_history)
-    return [note_history]
-
-
-def chat(message, history):
-    history = history or []
-    if history: 
-        history_useful = ['</s> <s>'.join([str(a[0])+'</s> <s>'+str(a[1]) for a in history])]
-    else:
-        history_useful = []
-    history_useful = add_note_to_history(message, history_useful)
-    inputs = tokenizer(history_useful, return_tensors="pt")
-    inputs, history_useful, history = take_last_tokens(inputs, history_useful, history)
-    reply_ids = model.generate(**inputs)
-    response = tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0]
-    history_useful = add_note_to_history(response, history_useful)
-    list_history = history_useful[0].split('</s> <s>')
-    history.append((list_history[-2], list_history[-1]))
-#    store_message(message, response) # Save to dataset - uncomment if you uncomment above to save inputs and outputs to your dataset
-    return history, history
+import pandas as pd
+import json
+from collections import defaultdict
+
+# Create tokenizer for biomed model
+from transformers import pipeline, AutoTokenizer, AutoModelForTokenClassification
+tokenizer = AutoTokenizer.from_pretrained("d4data/biomedical-ner-all")
+model = AutoModelForTokenClassification.from_pretrained("d4data/biomedical-ner-all")
+pipe = pipeline("ner", model=model, tokenizer=tokenizer, aggregation_strategy="simple")
+
+# Matplotlib for entity graph
+import matplotlib.pyplot as plt
+plt.switch_backend("Agg")
+
+# Load examples from JSON
+EXAMPLES = {}
+with open("examples.json", "r") as f:
+    example_json = json.load(f)
+    EXAMPLES = {x["text"]: x["label"] for x in example_json}
+
+def group_by_entity(raw):
+    out = defaultdict(int)
+    for ent in raw:
+        out[ent["entity_group"]] += 1
+    # out["total"] = sum(out.values())
+    return out
+
+
+def plot_to_figure(grouped):
+    fig = plt.figure()
+    plt.bar(x=list(grouped.keys()), height=list(grouped.values()))
+    plt.margins(0.2)
+    plt.subplots_adjust(bottom=0.4)
+    plt.xticks(rotation=90)
+    return fig
+
+
+def ner(text):
+    raw = pipe(text)
+    ner_content = {
+        "text": text,
+        "entities": [
+            {
+                "entity": x["entity_group"],
+                "word": x["word"],
+                "score": x["score"],
+                "start": x["start"],
+                "end": x["end"],
+            }
+            for x in raw
+        ],
+    }
     
+    grouped = group_by_entity(raw)
+    figure = plot_to_figure(grouped)
+    label = EXAMPLES.get(text, "Unknown")
 
-SAMPLE_RATE = 16000
-model = nemo_asr.models.EncDecRNNTBPEModel.from_pretrained("nvidia/stt_en_conformer_transducer_xlarge")
-model.change_decoding_strategy(None)
-model.eval()
-
-def process_audio_file(file):
-    data, sr = librosa.load(file)
-    if sr != SAMPLE_RATE:
-        data = librosa.resample(data, orig_sr=sr, target_sr=SAMPLE_RATE)
-    # monochannel
-    data = librosa.to_mono(data)
-    return data
+    meta = {
+        "entity_counts": grouped,
+        "entities": len(set(grouped.keys())),
+        "counts": sum(grouped.values()),
+    }
 
+    return (ner_content, meta, label, figure)
 
-def transcribe(audio, state = ""):   
-    if state is None:
-        state = ""
-    audio_data = process_audio_file(audio)
-    with tempfile.TemporaryDirectory() as tmpdir:
-        audio_path = os.path.join(tmpdir, f'audio_{uuid.uuid4()}.wav')
-        soundfile.write(audio_path, audio_data, SAMPLE_RATE)
-        transcriptions = model.transcribe([audio_path])
-        if type(transcriptions) == tuple and len(transcriptions) == 2:
-            transcriptions = transcriptions[0]
-        transcriptions = transcriptions[0]
-#    store_message(transcriptions, state) # Save to dataset - uncomment to store into a dataset - hint you will need your HF_TOKEN
-    state = state + transcriptions + " "
-    return state, state
 
-iface = gr.Interface(
-    fn=transcribe,
-    inputs=[
-        gr.Audio(source="microphone", type='filepath', streaming=True),
-        "state",
-    ],
+interface = gr.Interface(
+    ner,
+    inputs=gr.Textbox(label="Note text", value=""),
     outputs=[
-        "textbox",
-        "state",
+        gr.HighlightedText(label="NER", combine_adjacent=True),
+        gr.JSON(label="Entity Counts"),
+        gr.Label(label="Rating"),
+        gr.Plot(label="Bar"),
     ],
-    layout="horizontal",
-    theme="huggingface",
-    title="🗣️LiveSpeechRecognition🧠Memory💾",
-    description=f"Live Automatic Speech Recognition (ASR) with Memory💾 Dataset.",
-    allow_flagging='never',
-    live=True,    
-#    article=f"Result Output Saved to Memory💾 Dataset: [{DATASET_REPO_URL}]({DATASET_REPO_URL})"
+    examples=list(EXAMPLES.keys()),
+    allow_flagging="never",
 )
-iface.launch()
+
+interface.launch()

examples.json

@@ -0,0 +1,18 @@
+[
+  {
+    "text": "In March and April the patient had two falls.  One was related to asthma, heart palpitations.  The second was due to syncope and post covid vaccination dizziness during exercise.  The patient is now getting an EKG.  Former EKG had shown that there was a bundle branch block.  Patient had some uncontrolled immune system reactions like anaphylaxis and shortness of breath.",
+    "label": 3
+  },
+  {
+    "text": "During the fertility engagement, there was alot of anxiety and we just felt depressed.  It was difficult but finally after nutrition advice and changing habits such as cutting out smoking and improving daily activities we felt less agitated.  Then our baby came.  When he was born in December he had some minor jaundice but nothing serious, yet we had problems sleeping.  Both of us felt so relieved to have had the help, and now that the baby doesn't cry all night, we feel less fatigue and can sleep through night.",
+    "label": 2
+  },
+  {
+    "text": "51 y/o male with history of skin cancer, asthma, allergic to NSAIDS, dogs, cats, mold.  Daily long distance swimming helped breathing problems.  COVID reactions were severe yet recovery was fastest when maintaining a healthy diet, low fat and chloresterol, as well as limiting sugar intake.",
+    "label": 0
+  },
+  {
+    "text": "SNOMEDCT 183452005 is an emergency hospital encounter.  CPT 99378 is an intervention with supervision of a hospice patient.  HCPCS T2046 Hospice long term care, room and board only; per diem.  LOINC for Care Plan Document Type is code 80748-7 - Clinical genetics Plan of care note.  LOINC Care Plan Document Type 80751-1 is a Dermatology Plan of care note.  ICD10CM M84.371A is a Stress fracture, right ankle, initial encounter for fracture which is a Condition/Diagnosis/Problem Lower Body Fracture.  The purpose of this value set is to represent concepts for a diagnosis of a fracture of a bone in the lower body.",
+    "label": 1
+  }
+]

requirements.txt

@@ -1,7 +1,71 @@
-nemo_toolkit[asr]
-transformers
-torch
-gradio
-Werkzeug
-huggingface_hub
-Pillow
+aiohttp==3.8.3
+aiosignal==1.2.0
+anyio==3.6.1
+async-timeout==4.0.2
+attrs==22.1.0
+bcrypt==4.0.1
+black==22.10.0
+certifi==2022.9.24
+cffi==1.15.1
+charset-normalizer==2.1.1
+click==8.1.3
+contourpy==1.0.5
+cryptography==38.0.1
+cycler==0.11.0
+fastapi==0.85.0
+ffmpy==0.3.0
+filelock==3.8.0
+fonttools==4.37.4
+frozenlist==1.3.1
+fsspec==2022.8.2
+gradio==3.4.1
+h11==0.12.0
+httpcore==0.15.0
+httpx==0.23.0
+huggingface-hub==0.10.0
+idna==3.4
+Jinja2==3.1.2
+kiwisolver==1.4.4
+linkify-it-py==1.0.3
+markdown-it-py==2.1.0
+MarkupSafe==2.1.1
+matplotlib==3.6.1
+mdit-py-plugins==0.3.1
+mdurl==0.1.2
+multidict==6.0.2
+mypy-extensions==0.4.3
+numpy==1.23.3
+orjson==3.8.0
+packaging==21.3
+pandas==1.5.0
+paramiko==2.11.0
+pathspec==0.10.1
+Pillow==9.2.0
+platformdirs==2.5.2
+pycparser==2.21
+pycryptodome==3.15.0
+pydantic==1.10.2
+pydub==0.25.1
+PyNaCl==1.5.0
+pyparsing==3.0.9
+python-dateutil==2.8.2
+python-multipart==0.0.5
+pytz==2022.4
+PyYAML==6.0
+regex==2022.9.13
+requests==2.28.1
+rfc3986==1.5.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.20.4
+tokenizers==0.12.1
+tomli==2.0.1
+torch==1.12.1
+tqdm==4.64.1
+transformers==4.22.2
+typing_extensions==4.4.0
+uc-micro-py==1.0.1
+urllib3==1.26.12
+uvicorn==0.18.3
+websockets==10.3
+yarl==1.8.1