all things

app.py

@@ -0,0 +1,98 @@
+import gradio as gr
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+# Load the model
+def load_model():
+    repo_id = "KolumbusLindh/LoRA-4100"
+    model_file = "unsloth.F16.gguf"
+    
+    local_path = hf_hub_download(repo_id=repo_id, filename=model_file)
+    print(f"Loading model from: {local_path}")
+    return Llama(model_path=local_path, n_ctx=2048, n_threads=8)
+
+print("Starting model loading...")
+model = load_model()
+print("Model loaded successfully!")
+
+# Function to generate and evaluate content
+def generate_and_evaluate(preconfigured_prompt):
+    # Step 1: Generate content
+    generation_prompt = [
+        {"role": "user", "content": preconfigured_prompt}
+    ]
+    generated_response = model.create_chat_completion(
+        messages=generation_prompt,
+        max_tokens=256,
+        temperature=1.5
+    )
+    generated_content = generated_response['choices'][0]['message']['content']
+    
+    # Step 2: Evaluate the generated content
+    evaluation_prompt = [
+        {"role": "system", "content": "You are a strict language evaluator who provides binary assessments of texts."},
+        {"role": "user", "content": f"""Carefully evaluate the generated story:
+Prompt: {preconfigured_prompt}
+Generated response: {generated_content}
+Provide a clear evaluation as follows:
+For each question, write the full question followed by your "Yes" or "No" answer.
+Example format:
+1. Is the story exactly 50 words? - Yes
+2. Does the story contain the letter 'a'? - No
+Now answer these questions:
+1. Is the story exactly 50 words?
+2. Does the story contain the letter 'a'?
+3. Does the story contain the word "alabaster"?
+4. Does the reader understand that the cat's name is Alabaster?
+5. Is the story 100% in English?
+6. Does the text rhyme?"""}
+    ]
+    evaluation_response = model.create_chat_completion(
+        messages=evaluation_prompt,
+        max_tokens=128,
+        temperature=0.2
+    )
+    evaluation_results = evaluation_response['choices'][0]['message']['content']
+    
+    return generated_content, evaluation_results
+
+# Preconfigured prompt
+PRECONFIGURED_PROMPT = """Write a story about the cat Alabaster. It should be exactly 50 words and you are not allowed to use the letter 'a'. The reader must understand that the cat's name is Alabaster. Only replacing the letter 'a' with something like "_" is not enough. The text should rhyme."""
+
+# Gradio interface
+with gr.Blocks(title="LLM as a Judge") as demo:
+    gr.Markdown("## LLM as a Judge 🧐")
+    
+    generate_evaluate_button = gr.Button("Judge the LLM!")
+
+    # Label for the preconfigured prompt
+    gr.Label("Preconfigured prompt:")
+    gr.Label(PRECONFIGURED_PROMPT)
+    
+    generated_output = gr.Textbox(
+        label="Generated Content", 
+        placeholder="The generated content will appear here...", 
+        lines=5, 
+        interactive=False
+    )
+    
+    evaluation_output = gr.Textbox(
+        label="Evaluation Results", 
+        placeholder="The evaluation results will appear here...", 
+        lines=8, 
+        interactive=False
+    )
+    
+    # Link generation and evaluation
+    generate_evaluate_button.click(
+        fn=generate_and_evaluate,
+        inputs=[gr.State(PRECONFIGURED_PROMPT)],
+        outputs=[generated_output, evaluation_output]
+    )
+
+# Launch the app
+demo.launch(
+    server_name="0.0.0.0",
+    server_port=7860,
+    share=False
+)

requirements.txt

@@ -0,0 +1,3 @@
+huggingface_hub==0.25.2
+gradio
+llama-cpp-python