Spaces:

bryanmildort
/

gpt-notes-summarizer-demo

Paused

bryanmildort commited on Mar 13, 2023

Commit

7dfdc23

1 Parent(s): 5e720dd

Update app.py

Files changed (1) hide show

app.py CHANGED Viewed

@@ -24,7 +24,7 @@ st.write(device_str)
 tokenizer = AutoTokenizer.from_pretrained("bryanmildort/gpt-clinical-notes-summarizer")
 checkpoint = "bryanmildort/gpt-clinical-notes-summarizer"
-model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map="auto", offload_folder="offload", offload_state_dict = True, torch_dtype=torch.float16)
 device_map = infer_auto_device_map(model, dtype="float16")
 st.write(device_map)

 tokenizer = AutoTokenizer.from_pretrained("bryanmildort/gpt-clinical-notes-summarizer")
 checkpoint = "bryanmildort/gpt-clinical-notes-summarizer"
+model = AutoModelForCausalLM.from_pretrained(checkpoint, device_map="auto", offload_folder="offload", offload_state_dict = True, load_in_8bit=True)
 device_map = infer_auto_device_map(model, dtype="float16")
 st.write(device_map)