Rename app.py to compress.py

app.py

@@ -1,19 +0,0 @@
-!pip install langchain
-import langchain
-
-import streamlit as st
-from langchain.embeddings import OpenAIEmbeddings
-
-# Load model from HF Spaces
-model = OpenAIEmbeddings(model_name="PyaeSoneK/legalQAcustom") 
-
-# Streamlit UI
-st.header("Seon's Legal QA for Dummies")
-input_text = st.text_input("Give me your wildest legal thoughts:")
-
-if st.button("Generate"):
-  # Get embeddings
-  embeddings = model(input_text)
-  
-  # Display results
-  st.write(embeddings)

compress.py

@@ -0,0 +1,18 @@
+from transformers import AutoModelForCausalLM
+import torch
+
+# Load model
+model = AutoModelForCausalLM.from_pretrained("PyaeSoneK/LlamaV2LegalFineTuned")
+
+# Compress model...
+
+# Pruning
+import torchprune as prune 
+pruned_model = prune.ln_structured(model, amount=0.3)
+
+# Quantization 
+from torchquant import quantize
+quantized_model = quantize(pruned_model, dtype=torch.qint8)
+
+# Export smaller model
+quantized_model.save_pretrained("/path/to/smaller_model")