Spaces:
Runtime error
Runtime error
File size: 4,753 Bytes
e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 |
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from llama_cpp import Llama
from concurrent.futures import ProcessPoolExecutor, as_completed
import uvicorn
from dotenv import load_dotenv
from difflib import SequenceMatcher
from tqdm import tqdm
import multiprocessing
load_dotenv()
app = FastAPI()
# Configuración de los modelos
models = [
{"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf"},
]
# Función para cargar un modelo
def load_model(model_config):
return Llama.from_pretrained(repo_id=model_config['repo_id'], filename=model_config['filename'])
# Cargar modelos en paralelo
def load_all_models():
with ProcessPoolExecutor() as executor:
future_to_model = {executor.submit(load_model, model): model for model in models}
loaded_models = {}
for future in as_completed(future_to_model):
model = future_to_model[future]
try:
loaded_models[model['repo_id']] = future.result()
print(f"Modelo cargado en RAM: {model['repo_id']}")
except Exception as exc:
print(f"Error al cargar modelo {model['repo_id']}: {exc}")
return loaded_models
# Cargar modelos en memoria
llms = load_all_models()
class ChatRequest(BaseModel):
message: str
top_k: int = 50
top_p: float = 0.95
temperature: float = 0.7
def generate_chat_response(request, llm):
try:
user_input = normalize_input(request.message)
response = llm.create_chat_completion(
messages=[{"role": "user", "content": user_input}],
top_k=request.top_k,
top_p=request.top_p,
temperature=request.temperature
)
reply = response['choices'][0]['message']['content']
return {"response": reply, "literal": user_input}
except Exception as e:
return {"response": f"Error: {str(e)}", "literal": user_input}
def normalize_input(input_text):
return input_text.strip()
def filter_duplicates(responses):
seen = set()
unique_responses = []
for response in responses:
lines = response.split('\n')
unique_lines = set()
for line in lines:
if line not in seen:
seen.add(line)
unique_lines.add(line)
unique_responses.append('\n'.join(unique_lines))
return unique_responses
def select_best_response(responses):
# Eliminar respuestas duplicadas
unique_responses = filter_duplicates(responses)
# Deduplicar respuestas
unique_responses = list(set(unique_responses))
# Filtrar respuestas coherentes
coherent_responses = filter_by_coherence(unique_responses)
# Seleccionar la mejor respuesta
best_response = filter_by_similarity(coherent_responses)
return best_response
def filter_by_coherence(responses):
# Implementa aquí un filtro de coherencia si es necesario
return responses
def filter_by_similarity(responses):
responses.sort(key=len, reverse=True)
best_response = responses[0]
for i in range(1, len(responses)):
ratio = SequenceMatcher(None, best_response, responses[i]).ratio()
if ratio < 0.9:
best_response = responses[i]
break
return best_response
@app.post("/generate_chat")
async def generate_chat(request: ChatRequest):
if not request.message.strip():
raise HTTPException(status_code=400, detail="The message cannot be empty.")
print(f"Procesando solicitud: {request.message}")
# Utilizar un ProcessPoolExecutor para procesar los modelos en paralelo
def worker_function(llm):
return generate_chat_response(request, llm)
with ProcessPoolExecutor() as executor:
futures = [executor.submit(worker_function, llm) for llm in llms.values()]
responses = []
for future in tqdm(as_completed(futures), total=len(futures), desc="Generando respuestas"):
response = future.result()
responses.append(response['response'])
print(f"Modelo procesado: {response['literal'][:30]}...")
# Seleccionar la mejor respuesta
best_response = select_best_response(responses)
print(f"Mejor respuesta seleccionada: {best_response}")
return {
"best_response": best_response,
"all_responses": responses
}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860)
|