llmlingua optimization

qa_engine/qa_engine.py

@@ -86,6 +86,9 @@ class TransformersPipelineModel(LLM):
         )
 
     def _call(self, prompt: str, stop: Optional[list[str]] = None) -> str:
+        from llmlingua import PromptCompressor
+        llm_lingua = PromptCompressor()
+        prompt = llm_lingua.compress_prompt(prompt, instruction="", question="", target_token=400)
         output_text = self.pipeline(prompt)[0]['generated_text']
         output_text = output_text.replace(prompt+'\n', '')
         return output_text