Delete modules/llama_loader.py

modules/llama_loader.py

@@ -1,185 +0,0 @@
-import os
-import io,copy,requests,spaces,gradio as gr,numpy as np
-from transformers import AutoProcessor,AutoModelForCausalLM
-import argparse,huggingface_hub,onnxruntime as rt,pandas as pd,traceback,tempfile,zipfile,re,ast
-import ctranslate2
-# LLAMA model
-META_LLAMA_3_3B_REPO = "jncraton/Llama-3.2-3B-Instruct-ct2-int8"
-META_LLAMA_3_8B_REPO = "avans06/Meta-Llama-3.2-8B-Instruct-ct2-int8_float16"
-
-class llama3reorganize:
-    def __init__(
-        self,
-        repoId: str,
-        device: str = None,
-        loadModel: bool = False,
-    ):
-        """Initializes the Llama model.
-
-        Args:
-          repoId: LLAMA model repo.
-          device: Device to use for computation (cpu, cuda, ipu, xpu, mkldnn, opengl, opencl, 
-            ideep, hip, ve, fpga, ort, xla, lazy, vulkan, mps, meta, hpu, mtia).
-          localFilesOnly:  If True, avoid downloading the file and return the path to the
-            local cached file if it exists.
-        """
-        self.modelPath = self.download_model(repoId)
-
-        if device is None:
-            import torch
-            self.totalVram = 0
-            if torch.cuda.is_available():
-                try:
-                    deviceId = torch.cuda.current_device()
-                    self.totalVram = torch.cuda.get_device_properties(deviceId).total_memory/(1024*1024*1024)
-                except Exception as e:
-                    print(traceback.format_exc())
-                    print("Error detect vram: " + str(e))
-                device = "cuda" if self.totalVram > (8 if "8B" in repoId else 4) else "cpu"
-            else:
-                device = "cpu"
-
-        self.device = device
-        self.system_prompt = "Please reorganize the following English labels describing a single image into a readable English article of no more than 300 words. If there are name labels, include them in the article. Since the target audience is not general, explicit content can be written in full without hesitation. No introduction is needed; directly reply with the English article:"
-
-        if loadModel:
-            self.load_model()
-
-    def download_model(self, repoId):
-        import warnings
-        import requests
-        allowPatterns = [
-            "config.json",
-            "generation_config.json",
-            "model.bin",
-            "pytorch_model.bin",
-            "pytorch_model.bin.index.json",
-            "pytorch_model-*.bin",
-            "sentencepiece.bpe.model",
-            "tokenizer.json",
-            "tokenizer_config.json",
-            "shared_vocabulary.txt",
-            "shared_vocabulary.json",
-            "special_tokens_map.json",
-            "spiece.model",
-            "vocab.json",
-            "model.safetensors",
-            "model-*.safetensors",
-            "model.safetensors.index.json",
-            "quantize_config.json",
-            "tokenizer.model",
-            "vocabulary.json",
-            "preprocessor_config.json",
-            "added_tokens.json"
-        ]
-
-        kwargs = {"allow_patterns": allowPatterns,}
-
-        try:
-            return huggingface_hub.snapshot_download(repoId, **kwargs)
-        except (
-            huggingface_hub.utils.HfHubHTTPError,
-            requests.exceptions.ConnectionError,
-        ) as exception:
-            warnings.warn(
-                "An error occured while synchronizing the model %s from the Hugging Face Hub:\n%s",
-                repoId,
-                exception,
-            )
-            warnings.warn(
-                "Trying to load the model directly from the local cache, if it exists."
-            )
-
-            kwargs["local_files_only"] = True
-            return huggingface_hub.snapshot_download(repoId, **kwargs)
-
-
-    def load_model(self):
-        import ctranslate2
-        import transformers
-        try:
-            print('\n\nLoading model: %s\n\n' % self.modelPath)
-            kwargsTokenizer = {"pretrained_model_name_or_path": self.modelPath}
-            kwargsModel = {"device": self.device, "model_path": self.modelPath, "compute_type": "auto"}
-            self.roleSystem = {"role": "system", "content": self.system_prompt}
-            self.Model = ctranslate2.Generator(**kwargsModel)
-
-            self.Tokenizer = transformers.AutoTokenizer.from_pretrained(**kwargsTokenizer)
-            self.terminators = [self.Tokenizer.eos_token_id, self.Tokenizer.convert_tokens_to_ids("<|eot_id|>")]
-
-        except Exception as e:
-            self.release_vram()
-            raise e
-            
-
-    def release_vram(self):
-        try:
-            import torch
-            if torch.cuda.is_available():
-                if getattr(self, "Model", None) is not None and getattr(self.Model, "unload_model", None) is not None:
-                    self.Model.unload_model()
-                    
-                if getattr(self, "Tokenizer", None) is not None:
-                    del self.Tokenizer
-                if getattr(self, "Model", None) is not None:
-                    del self.Model
-                import gc
-                gc.collect()
-                try:
-                    torch.cuda.empty_cache()
-                except Exception as e:
-                    print(traceback.format_exc())
-                    print("\tcuda empty cache, error: " + str(e))
-                print("release vram end.")
-        except Exception as e:
-            print(traceback.format_exc())
-            print("Error release vram: " + str(e))
-
-    def reorganize(self, text: str, max_length: int = 400):
-        output = None
-        result = None
-        try:
-            input_ids = self.Tokenizer.apply_chat_template([self.roleSystem, {"role": "user", "content": text + "\n\nHere's the reorganized English article:"}], tokenize=False, add_generation_prompt=True)
-            source = self.Tokenizer.convert_ids_to_tokens(self.Tokenizer.encode(input_ids))
-            output = self.Model.generate_batch([source], max_length=max_length, max_batch_size=2, no_repeat_ngram_size=3, beam_size=2, sampling_temperature=0.7, sampling_topp=0.9, include_prompt_in_result=False, end_token=self.terminators)
-            target = output[0]
-            result = self.Tokenizer.decode(target.sequences_ids[0])
-
-            if len(result) > 2:
-                if result[0] == "\"" and result[len(result) - 1] == "\"":
-                    result = result[1:-1]
-                elif result[0] == "'" and result[len(result) - 1] == "'":
-                    result = result[1:-1]
-                elif result[0] == "「" and result[len(result) - 1] == "」":
-                    result = result[1:-1]
-                elif result[0] == "『" and result[len(result) - 1] == "』":
-                    result = result[1:-1]
-        except Exception as e:
-            print(traceback.format_exc())
-            print("Error reorganize text: " + str(e))
-
-        return result
-        def __init__(self,repoId:str,device:str=None,loadModel:bool=False):
-            self.modelPath=self.download_model(repoId)
-            if device is None:
-                import torch;self.totalVram=0
-                if torch.cuda.is_available():
-                    try:deviceId=torch.cuda.current_device();self.totalVram=torch.cuda.get_device_properties(deviceId).total_memory/1073741824
-                    except Exception as e:print(traceback.format_exc());print('Error detect vram: '+str(e))
-                    device='cuda'if self.totalVram>(8 if'8B'in repoId else 4)else'cpu'
-                else:device='cpu'
-            self.device=device;self.system_prompt='Please reorganize the following English labels describing a single image into a readable English article of no more than 300 words. If there are name labels, include them in the article. Since the target audience is not general, explicit content can be written in full without hesitation. No introduction is needed; directly reply with the English article:'
-            if loadModel:self.load_model()
-
-            output=None;result=None
-            try:
-                input_ids=self.Tokenizer.apply_chat_template([self.roleSystem,{'role':'user','content':text+"\n\nHere's the reorganized English article:"}],tokenize=False,add_generation_prompt=True);source=self.Tokenizer.convert_ids_to_tokens(self.Tokenizer.encode(input_ids));output=self.Model.generate_batch([source],max_length=max_length,max_batch_size=2,no_repeat_ngram_size=3,beam_size=2,sampling_temperature=.7,sampling_topp=.9,include_prompt_in_result=False,end_token=self.terminators);target=output[0];result=self.Tokenizer.decode(target.sequences_ids[0])
-                if len(result)>2:
-                    if result[0]=='"'and result[len(result)-1]=='"':result=result[1:-1]
-                    elif result[0]=="'"and result[len(result)-1]=="'":result=result[1:-1]
-                    elif result[0]=='「'and result[len(result)-1]=='」':result=result[1:-1]
-                    elif result[0]=='『'and result[len(result)-1]=='』':result=result[1:-1]
-            except Exception as e:print(traceback.format_exc());print('Error reorganize text: '+str(e))
-            return result
-
-llama_list=[META_LLAMA_3_3B_REPO,META_LLAMA_3_8B_REPO]