Compile the model for lower latency

app.py

@@ -219,11 +219,13 @@ def main():
 
     model = get_model(model_name, trust_remote_code)
     index = get_index(dir, search_time_s)
+
     if torch.cuda.is_available():
         model = model.half().cuda() if fp16 else model.bfloat16().cuda()
         # TODO: if huggingface datasets exposes an fp16 gpu option, use it here
     elif fp16:
         print('warning: used "FP16" on CPU-only system, ignoring...', file=stderr)
+    model.compile(mode="reduce-overhead")
 
     # function signature: (expanded tuple of input batches) -> tuple of output batches
     def search(query: list[str]) -> tuple[list[str]]: