import gradio as gr import os import base64 from io import BytesIO from mistralai import Mistral from pydantic import BaseModel, Field from datasets import load_dataset from PIL import Image import json import sqlite3 from datetime import datetime # Load the dataset ds = load_dataset("svjack/pokemon-blip-captions-en-zh") ds = ds["train"] # Load environment variables api_key = os.environ.get('MISTRAL_API_KEY') if not api_key: raise ValueError("MISTRAL_API_KEY is not set in the environment variables.") # Create sample history hist = [str({"en": ds[i]["en_text"], "zh": ds[i]["zh_text"]}) for i in range(8)] hist_str = "\n".join(hist) # Define the Caption model class Caption(BaseModel): en: str = Field(..., description="English caption of image", max_length=84) zh: str = Field(..., description="Chinese caption of image", max_length=64) # Initialize the Mistral client client = Mistral(api_key=api_key) def generate_caption(image): # Convert image to base64 buffered = BytesIO() image.save(buffered, format="JPEG") base64_image = base64.b64encode(buffered.getvalue()).decode('utf-8') messages = [ { "role": "system", "content": f''' You are a highly accurate image to caption transformer. Describe the image content in English and Chinese respectively. Make sure to FOCUS on item CATEGORY and COLOR! Do NOT provide NAMES! KEEP it SHORT! While adhering to the following JSON schema: {Caption.model_json_schema()} Following are some samples you should adhere to for style and tone: {hist_str} ''' }, { "role": "user", "content": [ { "type": "text", "text": "Describe the image in English and Chinese" }, { "type": "image_url", "image_url": f"data:image/jpeg;base64,{base64_image}" } ] } ] chat_response = client.chat.complete( model="pixtral-12b-2409", messages=messages, response_format = { "type": "json_object", } ) response_content = chat_response.choices[0].message.content try: caption_dict = json.loads(response_content) return Caption(**caption_dict) except json.JSONDecodeError as e: print(f"Error decoding JSON: {e}") return None # Initialize SQLite database def init_db(): conn = sqlite3.connect('feedback.db') c = conn.cursor() c.execute('''CREATE TABLE IF NOT EXISTS thumbs_up (id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT, input_data TEXT, output_data TEXT)''') conn.commit() conn.close() init_db() def process_image(image): if image is None: return "Please upload an image first." result = generate_caption(image) if result: return f"English caption: {result.en}\nChinese caption: {result.zh}" else: return "Failed to generate caption. Please check the API call or network connectivity." def thumbs_up(image, caption): # Convert image to base64 string for storage buffered = BytesIO() image.save(buffered, format="JPEG") img_str = base64.b64encode(buffered.getvalue()).decode() conn = sqlite3.connect('feedback.db') c = conn.cursor() c.execute("INSERT INTO thumbs_up (timestamp, input_data, output_data) VALUES (?, ?, ?)", (datetime.now().isoformat(), img_str, caption)) conn.commit() conn.close() print(f"Thumbs up data saved to database.") return gr.Info("Thank you for your feedback!") with gr.Blocks() as iface: gr.Markdown("# Image Captioner") gr.Markdown("Upload an image to generate captions in English and Chinese. Use the 'Thumbs Up' button if you like the result!") with gr.Row(): with gr.Column(scale=1): input_image = gr.Image(type="pil") with gr.Row(): clear_btn = gr.Button("Clear") submit_btn = gr.Button("Submit") with gr.Column(scale=1): output_text = gr.Textbox() thumbs_up_btn = gr.Button("Thumbs Up") clear_btn.click(fn=lambda: None, inputs=None, outputs=input_image) submit_btn.click(fn=process_image, inputs=input_image, outputs=output_text) thumbs_up_btn.click(fn=thumbs_up, inputs=[input_image, output_text], outputs=None) # Launch the interface iface.launch(share=True)